@smoothbricks/nx-plugin 0.7.5 → 0.7.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/README.md +20 -3
  2. package/dist/.build-inputs.json +12 -6
  3. package/dist/executors/bounded-exec/cpu-pool-thread.d.ts +3 -2
  4. package/dist/executors/bounded-exec/cpu-pool-thread.d.ts.map +1 -1
  5. package/dist/executors/bounded-exec/cpu-pool-thread.js +1 -1
  6. package/dist/executors/bounded-exec/cpu-pool.d.ts +24 -10
  7. package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -1
  8. package/dist/executors/bounded-exec/cpu-pool.js +96 -52
  9. package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
  10. package/dist/executors/bounded-exec/executor.js +115 -62
  11. package/dist/executors/bounded-exec/host-memory.d.ts +54 -0
  12. package/dist/executors/bounded-exec/host-memory.d.ts.map +1 -0
  13. package/dist/executors/bounded-exec/host-memory.darwin.d.ts +3 -0
  14. package/dist/executors/bounded-exec/host-memory.darwin.d.ts.map +1 -0
  15. package/dist/executors/bounded-exec/host-memory.darwin.js +115 -0
  16. package/dist/executors/bounded-exec/host-memory.js +29 -0
  17. package/dist/executors/bounded-exec/host-memory.linux.d.ts +14 -0
  18. package/dist/executors/bounded-exec/host-memory.linux.d.ts.map +1 -0
  19. package/dist/executors/bounded-exec/host-memory.linux.js +137 -0
  20. package/dist/executors/bounded-exec/leaf-peak.d.ts +56 -0
  21. package/dist/executors/bounded-exec/leaf-peak.d.ts.map +1 -0
  22. package/dist/executors/bounded-exec/leaf-peak.js +147 -0
  23. package/dist/executors/bounded-exec/mbx-reservations.d.ts +22 -0
  24. package/dist/executors/bounded-exec/mbx-reservations.d.ts.map +1 -1
  25. package/dist/executors/bounded-exec/mbx-reservations.js +22 -0
  26. package/dist/executors/bounded-exec/peak-waiter.d.ts +2 -0
  27. package/dist/executors/bounded-exec/peak-waiter.d.ts.map +1 -0
  28. package/dist/executors/bounded-exec/peak-waiter.js +79 -0
  29. package/dist/executors/bounded-exec/pool-admission.d.ts +38 -0
  30. package/dist/executors/bounded-exec/pool-admission.d.ts.map +1 -0
  31. package/dist/executors/bounded-exec/pool-admission.js +31 -0
  32. package/dist/executors/bounded-exec/run-cpu-pool.d.ts +6 -4
  33. package/dist/executors/bounded-exec/run-cpu-pool.d.ts.map +1 -1
  34. package/dist/executors/bounded-exec/run-cpu-pool.js +20 -7
  35. package/dist/executors/bounded-exec/verdict.d.ts +18 -0
  36. package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
  37. package/dist/executors/bounded-exec/verdict.js +21 -1
  38. package/managed/templates/github/actions/setup-devenv/action.yml +31 -2
  39. package/package.json +2 -2
  40. package/src/executors/bounded-exec/cpu-pool-thread.ts +4 -3
  41. package/src/executors/bounded-exec/cpu-pool.ts +111 -49
  42. package/src/executors/bounded-exec/executor.ts +121 -59
  43. package/src/executors/bounded-exec/host-memory.darwin.ts +147 -0
  44. package/src/executors/bounded-exec/host-memory.linux.ts +147 -0
  45. package/src/executors/bounded-exec/host-memory.ts +77 -0
  46. package/src/executors/bounded-exec/leaf-peak.ts +152 -0
  47. package/src/executors/bounded-exec/mbx-reservations.ts +30 -0
  48. package/src/executors/bounded-exec/peak-waiter.ts +115 -0
  49. package/src/executors/bounded-exec/pool-admission.ts +64 -0
  50. package/src/executors/bounded-exec/run-cpu-pool.ts +20 -6
  51. package/src/executors/bounded-exec/verdict.ts +30 -1
  52. package/test-timings.json +136 -132
@@ -1,7 +1,7 @@
1
1
  import { type ChildProcess, type IOType, spawn } from 'node:child_process';
2
2
  import { accessSync, constants } from 'node:fs';
3
3
  import { mkdtemp, rm } from 'node:fs/promises';
4
- import { availableParallelism, userInfo } from 'node:os';
4
+ import { availableParallelism, constants as osConstants, userInfo } from 'node:os';
5
5
  import { delimiter, isAbsolute, join, resolve as resolvePath } from 'node:path';
6
6
  import { createInterface } from 'node:readline';
7
7
  import { Duplex, Readable } from 'node:stream';
@@ -36,6 +36,7 @@ import {
36
36
  import { GATEWAY_SOCKET } from './gateway-lease.js';
37
37
  import type { HangDiagnostics } from './hang-diagnostics.js';
38
38
  import { captureHangDiagnostics, HANG_CAPTURE_MS } from './hang-diagnostics.js';
39
+ import { type LeafPeak, PEAK_FD, peakWaiter, readWaiterReport, underWaiter } from './leaf-peak.js';
39
40
  import type { CpuWaitRow } from './lease-wait.js';
40
41
  import type { LockWait } from './lock-wait.js';
41
42
  import {
@@ -48,7 +49,7 @@ import {
48
49
  withMbxLeaf,
49
50
  } from './mbx-pool.js';
50
51
  import { MBX_QUEUE_POLL_MS, type MbxQueuedRequest, snapshotMbxQueue } from './mbx-queue.js';
51
- import { RESERVATIONS_ENV, type Reservation, reserveCores } from './mbx-reservations.js';
52
+ import { describeUnserved, RESERVATIONS_ENV, type Reservation, reserveCores, runEndpoint } from './mbx-reservations.js';
52
53
  import { createNoSpaceWatch } from './no-space.js';
53
54
  import {
54
55
  COWSHED_EXECUTABLE_ENV,
@@ -92,6 +93,7 @@ import {
92
93
  judgeRun,
93
94
  type NoSpaceSighting,
94
95
  readJunitReport,
96
+ readLastPeak,
95
97
  taskDirectory,
96
98
  withJunitReport,
97
99
  writeRecord,
@@ -103,12 +105,6 @@ type IdleDiagnosticResult =
103
105
  | { kind: 'pending'; startedAtMs: number | null }
104
106
  | { kind: 'captured'; snapshot: HangDiagnostics }
105
107
  | { kind: 'failed'; detail: string };
106
- const EXIT_CODE_BY_SIGNAL: Partial<Record<NodeJS.Signals, number>> = {
107
- SIGHUP: 129,
108
- SIGINT: 130,
109
- SIGTERM: 143,
110
- };
111
-
112
108
  /** The pool descriptor: only a grant may give it, so neither the target's nor the inherited environment passes it on. */
113
109
  const POOL_ENV = [CPU_TOKENS_ENV, CPU_POOL_ENV];
114
110
  /**
@@ -319,6 +315,8 @@ export async function runBoundedExec(
319
315
  ): Promise<BoundedExecResult> {
320
316
  const cwd = resolveCwd(options.cwd, context.root);
321
317
  const directory = context.record?.directory ?? null;
318
+ // The task's last recorded peak is its charge in its parent's pool, read before the clear below removes it.
319
+ const lastPeak = directory !== null && inherited !== null ? await readLastPeak(directory) : null;
322
320
  // A previous run's verdict and report go first, so neither can be read as this run's.
323
321
  if (directory !== null) {
324
322
  await clearTaskDirectory(directory);
@@ -366,6 +364,20 @@ export async function runBoundedExec(
366
364
  // Nobody is left to tell.
367
365
  }
368
366
  };
367
+ // A run refused before its command exists says why and records an operational failure, and no peak: nothing ran.
368
+ const refuseUnstarted = async (message: string): Promise<BoundedExecResult> => {
369
+ appendStderr(message);
370
+ if (context.record !== undefined) {
371
+ const { task, directory: recordDirectory } = context.record;
372
+ await writeRecord(recordDirectory, {
373
+ task: task.id,
374
+ hash: task.hash,
375
+ verdict: judgeOperationalFailure(1, null),
376
+ peakBytes: null,
377
+ });
378
+ }
379
+ return { success: false, terminalOutput: outputChunks.join('') };
380
+ };
369
381
 
370
382
  // The fixture pair names one segment: a fixture with no build budget, or a budget with no fixture, is a
371
383
  // target that does not say what it runs, so it does not run. Units are counted at the end of that segment.
@@ -374,14 +386,9 @@ export async function runBoundedExec(
374
386
  (options.unitsCommand !== undefined && options.fixtureCommand === undefined) ||
375
387
  (fixture !== null && !(fixture.timeoutMs > 0))
376
388
  ) {
377
- appendStderr(
389
+ return refuseUnstarted(
378
390
  'bounded-exec: fixtureCommand and a positive fixtureTimeoutMs must be named together, and unitsCommand only with them; refusing to run the command\n',
379
391
  );
380
- if (context.record !== undefined) {
381
- const { task, directory: recordDirectory } = context.record;
382
- await writeRecord(recordDirectory, { task: task.id, hash: task.hash, verdict: judgeOperationalFailure(1, null) });
383
- }
384
- return { success: false, terminalOutput: outputChunks.join('') };
385
392
  }
386
393
 
387
394
  // A leg that runs bun runs the Bun its workspace declares, or it does not run. This comes before the RAM lease and the
@@ -394,13 +401,15 @@ export async function runBoundedExec(
394
401
  pathSource: options.env?.PATH === undefined ? 'the inherited environment' : "the target's env",
395
402
  });
396
403
  if (runtime !== null) {
397
- appendStderr(describeRuntimePinRefusal(runtime, context.record?.task.id));
398
- if (context.record !== undefined) {
399
- const { task, directory: recordDirectory } = context.record;
400
- await writeRecord(recordDirectory, { task: task.id, hash: task.hash, verdict: judgeOperationalFailure(1, null) });
401
- }
402
- return { success: false, terminalOutput: outputChunks.join('') };
404
+ return refuseUnstarted(describeRuntimePinRefusal(runtime, context.record?.task.id));
403
405
  }
406
+ if (lastPeak !== null && !lastPeak.ok) {
407
+ return refuseUnstarted(
408
+ `bounded-exec: ${lastPeak.error}; it is this task's memory charge, so the command does not run\n`,
409
+ );
410
+ }
411
+ // A task with no recorded peak is charged nothing (null) until it has one.
412
+ const charge = lastPeak?.ok === true ? lastPeak.peakBytes : null;
404
413
 
405
414
  let lease: RamTempLease | null = null;
406
415
  let ordinaryTemp: string | null = null;
@@ -548,7 +557,7 @@ export async function runBoundedExec(
548
557
  let parentRefusal: string | null = null;
549
558
  let parentWait: CpuWaitRow | undefined;
550
559
  if (inherited !== null) {
551
- const fromParent = await takeParentCpuTokens(inherited, demand.want);
560
+ const fromParent = await takeParentCpuTokens(inherited, demand.want, charge);
552
561
  if (fromParent.granted) {
553
562
  grant = fromParent;
554
563
  } else if (fromParent.cause === 'absent') {
@@ -569,8 +578,13 @@ export async function runBoundedExec(
569
578
  sized = sizeRunner(demand, requested, grant.tokens);
570
579
  cpu = { tokens: grant.tokens, want: demand.want, waitedMs };
571
580
  appendStderr(
572
- `cowshed: cpu-pool wait done elapsed=${waitedMs}ms status=ok tokens=${grant.tokens}/${demand.want} runner=${demand.kind}\n`,
581
+ `cowshed: cpu-pool wait done elapsed=${waitedMs}ms status=ok tokens=${grant.tokens}/${demand.want} runner=${demand.kind} memory=${charge ?? 'none'}\n`,
573
582
  );
583
+ if (grant.oversized !== null) {
584
+ appendStderr(
585
+ `cowshed: cpu-pool granted while nothing else holds memory: this task's last peak of ${charge} bytes alone exceeds the run's ${grant.oversized.memory} bytes\n`,
586
+ );
587
+ }
574
588
  } else if (leaf?.kind === 'reserve') {
575
589
  sized = { command: sizeRunnerToGrant(leaf.demand, requested), env: {} };
576
590
  } else if (leaf !== null) {
@@ -633,6 +647,7 @@ export async function runBoundedExec(
633
647
  operational: boolean,
634
648
  noSpace: NoSpaceSighting | null,
635
649
  lockWaits: readonly LockWait[],
650
+ peakBytes: number | null,
636
651
  ): Promise<void> => {
637
652
  if (context.record === undefined) {
638
653
  return;
@@ -678,6 +693,7 @@ export async function runBoundedExec(
678
693
  verdict,
679
694
  ...(cpu ? { cpu } : {}),
680
695
  ...(parentWait ? { cpuWait: parentWait } : {}),
696
+ peakBytes,
681
697
  });
682
698
  };
683
699
  // A run refused before its command starts says why, gives back its grant and RAM, and records the failure.
@@ -686,25 +702,27 @@ export async function runBoundedExec(
686
702
  await releaseCpu();
687
703
  // The command never started, so its lease holds nothing of its own to keep.
688
704
  await releaseLease(false);
689
- await recordRun(1, null, Date.now() - asked, false, null, []);
705
+ await recordRun(1, null, Date.now() - asked, false, null, [], null);
690
706
  return { success: false, terminalOutput: outputChunks.join('') };
691
707
  };
692
708
  if (parentRefusal !== null) {
693
709
  return refuse(`cowshed: cpu-pool parent pool refused: ${parentRefusal}; refusing to run the command\n`);
694
710
  }
695
711
  // A reserved leaf takes its cores from its Nx run's mbx reservation endpoint (mbx-reservations.ts). A run that serves
696
- // none said why in its own output; the leaf does not run beside the pool.
697
- const reserved = leaf?.kind === 'reserve' ? { leaf, endpoint: process.env[RESERVATIONS_ENV] ?? '' } : null;
698
- if (reserved !== null && reserved.endpoint === '') {
712
+ // none said why in its own output, which a gate's failures-only output never shows, so the refusal says it too; the
713
+ // leaf does not run beside the pool.
714
+ const endpoint = leaf?.kind === 'reserve' ? runEndpoint(process.env) : null;
715
+ if (endpoint !== null && endpoint.kind !== 'served') {
699
716
  return refuse(
700
- `cowshed: a reserved leaf takes its cores from its Nx run's mbx reservation endpoint, and ${RESERVATIONS_ENV} names none (the run's preTasksExecution serves it); refusing to run the command\n`,
717
+ `cowshed: a reserved leaf takes its cores from its Nx run's mbx reservation endpoint, and ${RESERVATIONS_ENV} names none: ${describeUnserved(endpoint)}; refusing to run the command\n`,
701
718
  );
702
719
  }
720
+ const reserved = leaf?.kind === 'reserve' && endpoint?.kind === 'served' ? { leaf, endpoint: endpoint.socket } : null;
703
721
 
704
722
  // A granted run serves a sub-pool of its own grant, so its descendants draw from that and never from above it.
705
723
  // A pool that cannot be set up refuses the run: no granted command runs beside its grant.
706
724
  if (grant !== null) {
707
- const created = await createCpuPool(grant.tokens, grant.closed);
725
+ const created = await createCpuPool(grant.tokens, grant.closed, null);
708
726
  if (!created.ok) {
709
727
  return refuse(`cowshed: cpu pool unavailable: ${created.error}; refusing to run the command\n`);
710
728
  }
@@ -756,38 +774,67 @@ export async function runBoundedExec(
756
774
  const grantSlots: IOType[] = leaf !== null ? ['ignore', 'pipe'] : [];
757
775
  const rootGrantSlot: IOType[] = leaf !== null ? ['pipe'] : [];
758
776
  const root = owner?.rootArgv(shellCommand);
777
+ // The command's own descriptors past stderr, from fd 3: the lifecycle channel, the subtree admission, mbx's grant.
778
+ const descriptors: IOType[] =
779
+ root === undefined
780
+ ? hooked === null && leaf === null
781
+ ? []
782
+ : [hooked === null ? 'ignore' : 'pipe', ...grantSlots]
783
+ : [hooked === null ? 'ignore' : 'pipe', 'pipe', ...rootGrantSlot];
784
+ // A run that records its verdict records its command's peak too: the command runs under a waiter that reports it.
785
+ const waiter = context.record === undefined ? null : peakWaiter();
786
+ const measured =
787
+ waiter?.kind === 'waiter'
788
+ ? underWaiter(waiter.runtime, root?.file ?? '/bin/sh', root?.args ?? ['-c', shellCommand], descriptors)
789
+ : null;
759
790
  let child: ChildProcess;
760
791
  const results = directory === null ? null : await createTestResultWriter(directory, { cwd, root: context.root });
761
792
  if (results !== null && !results.ok) return refuse(`bounded-exec: ${results.error}\n`);
762
793
  resultWriter = results?.ok ? results.writer : null;
763
794
  try {
764
795
  // fd 3 is the runner lifecycle's duplex socket when the command is hooked; the subtree admission takes fd 4, which
765
- // cowshed closes fd 4 before exec; fd 5 is a Cargo leaf's mbx admission stream or a reserved leaf's ask and grant.
796
+ // cowshed closes fd 4 before exec; fd 5 is a Cargo leaf's mbx admission stream or a reserved leaf's ask and grant;
797
+ // the waiter, when there is one, passes them on and reports the command's peak on fd 6.
766
798
  child =
767
- root === undefined
768
- ? spawn(shellCommand, [], {
799
+ measured !== null
800
+ ? spawn(measured.file, measured.args, {
769
801
  cwd,
770
802
  env: mergeEnv(lease?.directory ?? ordinaryTemp, env),
771
- shell: true,
772
- stdio:
773
- hooked === null && leaf === null
774
- ? 'pipe'
775
- : ['pipe', 'pipe', 'pipe', hooked === null ? 'ignore' : 'pipe', ...grantSlots],
803
+ stdio: measured.stdio,
776
804
  detached: process.platform !== 'win32',
777
805
  windowsHide: true,
778
806
  })
779
- : spawn(root.file, root.args, {
780
- cwd,
781
- env: mergeEnv(lease?.directory ?? ordinaryTemp, env),
782
- stdio: ['pipe', 'pipe', 'pipe', hooked === null ? 'ignore' : 'pipe', 'pipe', ...rootGrantSlot],
783
- detached: process.platform !== 'win32',
784
- windowsHide: true,
785
- });
807
+ : root === undefined
808
+ ? spawn(shellCommand, [], {
809
+ cwd,
810
+ env: mergeEnv(lease?.directory ?? ordinaryTemp, env),
811
+ shell: true,
812
+ stdio: descriptors.length === 0 ? 'pipe' : ['pipe', 'pipe', 'pipe', ...descriptors],
813
+ detached: process.platform !== 'win32',
814
+ windowsHide: true,
815
+ })
816
+ : spawn(root.file, root.args, {
817
+ cwd,
818
+ env: mergeEnv(lease?.directory ?? ordinaryTemp, env),
819
+ stdio: ['pipe', 'pipe', 'pipe', ...descriptors],
820
+ detached: process.platform !== 'win32',
821
+ windowsHide: true,
822
+ });
786
823
  } catch (error) {
787
824
  // Nothing started, so nothing runs under the grant: it goes back here, with the RAM lease.
788
825
  return refuse(`cowshed: could not start the command: ${error instanceof Error ? error.message : String(error)}\n`);
789
826
  }
790
827
  subtreeRetired = false;
828
+ const reportChannel = child.stdio.at(PEAK_FD);
829
+ const waiterReport =
830
+ measured === null ? null : readWaiterReport(reportChannel instanceof Readable ? reportChannel : null);
831
+ // The leaf's runner is its command: under a waiter, which leads the group, the child the waiter names; else the
832
+ // group's leader itself. A hang snapshot names and samples it (specs/cowshed/08_testing.md).
833
+ const runnerPid: Promise<number | null> = waiterReport?.started ?? Promise.resolve(child.pid ?? null);
834
+ let runner: number | null = null;
835
+ void runnerPid.then((pid) => {
836
+ runner = pid;
837
+ });
791
838
  if (root !== undefined) {
792
839
  const channel = child.stdio[HANDLE_FD];
793
840
  const leased = lease;
@@ -847,22 +894,24 @@ export async function runBoundedExec(
847
894
  captureTimer = setTimeout(
848
895
  () => {
849
896
  if (state.settled || state.expiry !== null || idleCapture !== capture) return;
850
- const pid = child.pid;
851
- if (pid === undefined) {
897
+ const group = child.pid;
898
+ if (group === undefined) {
852
899
  idleCapture.result = { kind: 'failed', detail: 'runner PID unavailable; no holder conclusion' };
853
900
  return;
854
901
  }
855
902
  idleCapture.result = { kind: 'pending', startedAtMs: Date.now() };
856
- void Promise.resolve()
857
- .then(() =>
858
- captureDiagnostics({
903
+ void runnerPid
904
+ .then((pid) => {
905
+ if (pid === null)
906
+ throw new Error('runner PID unavailable: its waiter named no command; no holder conclusion');
907
+ return captureDiagnostics({
859
908
  pid,
860
- pgid: process.platform === 'win32' ? null : pid,
909
+ pgid: process.platform === 'win32' ? null : group,
861
910
  expiresAtMs: capture.expiresAtMs,
862
911
  deadlineMs: Math.max(0, Math.min(HANG_CAPTURE_MS, capture.expiresAtMs - Date.now())),
863
912
  signal: capture.controller.signal,
864
- }),
865
- )
913
+ });
914
+ })
866
915
  .then(
867
916
  (snapshot) => {
868
917
  if (idleCapture === capture) idleCapture.result = { kind: 'captured', snapshot };
@@ -1115,6 +1164,7 @@ export async function runBoundedExec(
1115
1164
  state.poolCloseError !== null || lifecycleFailure !== null || state.retireError !== null,
1116
1165
  state.noSpace === null ? null : await state.noSpace,
1117
1166
  state.lockWaits,
1167
+ null,
1118
1168
  ),
1119
1169
  )
1120
1170
  .finally(() => process.kill(process.pid, signal));
@@ -1183,7 +1233,7 @@ export async function runBoundedExec(
1183
1233
  const observed = idleCapture?.result;
1184
1234
  state.lockWaits = observed?.kind === 'captured' ? observed.snapshot.lockWaits : [];
1185
1235
  appendStderr(
1186
- `Hang diagnostics: runner pid=${child.pid ?? 'unavailable'} pgid=${
1236
+ `Hang diagnostics: runner pid=${runner ?? 'unavailable'} pgid=${
1187
1237
  process.platform === 'win32' ? 'unavailable' : (child.pid ?? 'unavailable')
1188
1238
  } expiryAtMs=${Date.now()} scheduledExpiryAtMs=${idleCapture?.expiresAtMs ?? 'unavailable'}\n`,
1189
1239
  );
@@ -1394,10 +1444,16 @@ export async function runBoundedExec(
1394
1444
 
1395
1445
  const code = exitCode ?? signalToExitCode(exitSignal);
1396
1446
  if (code !== 0 && state.expiry === null) {
1447
+ // No bound of this executor ended it, so a signal it died of came from outside (the kernel's OOM killer sends
1448
+ // SIGKILL): the signal is named, never folded into a bare status.
1449
+ const ended =
1450
+ exitCode === null && exitSignal !== null
1451
+ ? `was killed by ${exitSignal} (status ${code})`
1452
+ : `exited with status ${code}`;
1397
1453
  appendStderr(
1398
1454
  fixture !== null && !mainStarted
1399
- ? `Fixture command exited with status ${code}: ${segment ?? fixture.command}\n`
1400
- : `Command exited with status ${code}: ${command}\n`,
1455
+ ? `Fixture command ${ended}: ${segment ?? fixture.command}\n`
1456
+ : `Command ${ended}: ${command}\n`,
1401
1457
  );
1402
1458
  }
1403
1459
 
@@ -1420,10 +1476,18 @@ export async function runBoundedExec(
1420
1476
  await releaseLease(code !== 0 || state.expiry !== null || state.poolCloseError !== null || lifecycleFailure !== null);
1421
1477
  releaseReservation();
1422
1478
 
1479
+ // The command's peak, as its waiter reported it once both ended; a peak that was not measured is said, never zero.
1480
+ const reported: LeafPeak | null =
1481
+ waiterReport === null ? (waiter?.kind === 'unmeasured' ? waiter : null) : await waiterReport.peak;
1482
+ if (reported?.kind === 'unmeasured') {
1483
+ appendStderr(`bounded-exec: this run's memory peak was not measured (${reported.reason}); its record names none\n`);
1484
+ }
1485
+ const peakBytes = reported?.kind === 'measured' ? reported.bytes : null;
1486
+
1423
1487
  // Broken resource ownership, a lost native lifecycle, or leftovers that could not be retired fail even if a shell
1424
1488
  // branch later exited 0: its verdict is an ordinary operational failure, never a successful or silently unbounded run.
1425
1489
  const operationalFailure = state.poolCloseError !== null || lifecycleFailure !== null || state.retireError !== null;
1426
- await recordRun(code, state.expiry, Date.now() - startedAt, operationalFailure, seen, state.lockWaits);
1490
+ await recordRun(code, state.expiry, Date.now() - startedAt, operationalFailure, seen, state.lockWaits, peakBytes);
1427
1491
  return {
1428
1492
  success: !operationalFailure && resultFailure === null && state.expiry === null && code === 0,
1429
1493
  terminalOutput: outputChunks.join(''),
@@ -1488,11 +1552,9 @@ function targetEnv(env: Record<string, string> | undefined, say: (text: string)
1488
1552
  return kept;
1489
1553
  }
1490
1554
 
1555
+ /** The status a shell gives a command a signal ended: 128 plus the signal's number (SIGKILL 137, SIGTERM 143). */
1491
1556
  function signalToExitCode(signal: NodeJS.Signals | null): number {
1492
- if (!signal) {
1493
- return 1;
1494
- }
1495
- return EXIT_CODE_BY_SIGNAL[signal] ?? 1;
1557
+ return signal === null ? 1 : 128 + osConstants.signals[signal];
1496
1558
  }
1497
1559
 
1498
1560
  /** Resolves when `promise` settles or `ms` passes, whichever is first, leaving no timer behind. */
@@ -0,0 +1,147 @@
1
+ import { execFile } from 'node:child_process';
2
+ import { promisify } from 'node:util';
3
+ import type { HostMemory, RunMemoryReading } from './host-memory.js';
4
+
5
+ /**
6
+ * macOS's memory mechanisms. A run's memory is the host's, `hw.memsize`: macOS confines no job to a memory limit of
7
+ * its own. Bun reports a leaf's `ru_maxrss` in bytes, as macOS does.
8
+ */
9
+
10
+ const run = promisify(execFile);
11
+ const HW_MEMSIZE = 'hw.memsize';
12
+
13
+ async function readRunMemory(): Promise<RunMemoryReading> {
14
+ let stdout: string;
15
+ try {
16
+ ({ stdout } = await run('sysctl', ['-n', HW_MEMSIZE]));
17
+ } catch (error) {
18
+ return {
19
+ ok: false,
20
+ error: {
21
+ operation: 'read-run-memory',
22
+ code: 'unreadable',
23
+ subject: HW_MEMSIZE,
24
+ cause: error instanceof Error ? error.message : String(error),
25
+ },
26
+ };
27
+ }
28
+ const value = stdout.trim();
29
+ const bytes = Number(value);
30
+ if (!/^\d+$/.test(value) || !Number.isSafeInteger(bytes) || bytes === 0) {
31
+ return {
32
+ ok: false,
33
+ error: {
34
+ operation: 'read-run-memory',
35
+ code: 'malformed',
36
+ subject: HW_MEMSIZE,
37
+ cause: `sysctl printed ${JSON.stringify(value)}, not a byte count`,
38
+ },
39
+ };
40
+ }
41
+ return { ok: true, memory: { bytes, source: HW_MEMSIZE } };
42
+ }
43
+
44
+ /**
45
+ * Bun's FFI, as the peak waiter reaches it. Node, which runs the executor, loads this module too and has no `bun:ffi`;
46
+ * only the waiter, on Bun, reads a tree's footprint, so the FFI loads on its first read, by a specifier the compiler
47
+ * does not resolve against Node's types.
48
+ */
49
+ const BUN_FFI = 'bun:ffi';
50
+ const LIBPROC = '/usr/lib/libproc.dylib';
51
+ /** `proc_pid_rusage`'s oldest flavor, whose fields every later one starts with. */
52
+ const RUSAGE_INFO_V0 = 0;
53
+ /** `rusage_info_v0`: a 16-byte uuid, then ten u64 counters; `ri_phys_footprint` is the eighth, u64 index 9. */
54
+ const RUSAGE_INFO_V0_WORDS = 12;
55
+ const PHYS_FOOTPRINT = 9;
56
+
57
+ interface BunFfi {
58
+ dlopen(path: string, symbols: Record<string, { args: string[]; returns: string }>): { symbols: unknown };
59
+ ptr(view: ArrayBufferView): number;
60
+ }
61
+
62
+ /** The two libproc calls a tree's footprint takes, each pointer a Bun FFI address. */
63
+ interface LibprocSymbols {
64
+ /** The pids whose parent is `pid`, written into `into` (`bytes` long): how many, or -1. */
65
+ proc_listchildpids(pid: number, into: number, bytes: number): number;
66
+ /** `pid`'s `rusage_info` of `flavor`, written into `into`: 0, or -1 once nothing it may read holds the pid. */
67
+ proc_pid_rusage(pid: number, flavor: number, into: number): number;
68
+ }
69
+
70
+ function isBunFfi(module: unknown): module is BunFfi {
71
+ return (
72
+ typeof module === 'object' &&
73
+ module !== null &&
74
+ 'dlopen' in module &&
75
+ typeof module.dlopen === 'function' &&
76
+ 'ptr' in module &&
77
+ typeof module.ptr === 'function'
78
+ );
79
+ }
80
+
81
+ function isLibprocSymbols(symbols: unknown): symbols is LibprocSymbols {
82
+ return (
83
+ typeof symbols === 'object' &&
84
+ symbols !== null &&
85
+ 'proc_listchildpids' in symbols &&
86
+ typeof symbols.proc_listchildpids === 'function' &&
87
+ 'proc_pid_rusage' in symbols &&
88
+ typeof symbols.proc_pid_rusage === 'function'
89
+ );
90
+ }
91
+
92
+ /** One process's children and footprint, as libproc reads them. */
93
+ interface Footprints {
94
+ children(pid: number): number[];
95
+ /** What `pid` holds, or 0 once it is gone (or, a setuid program, unreadable to its owner's uid). */
96
+ footprint(pid: number): number;
97
+ }
98
+
99
+ let footprints: Promise<Footprints> | undefined;
100
+
101
+ async function loadFootprints(): Promise<Footprints> {
102
+ const ffi: unknown = await import(BUN_FFI);
103
+ if (!isBunFfi(ffi)) {
104
+ throw new Error(`${BUN_FFI} names no dlopen and ptr: a tree's footprint is read on Bun, by the peak waiter`);
105
+ }
106
+ const { symbols } = ffi.dlopen(LIBPROC, {
107
+ proc_listchildpids: { args: ['i32', 'ptr', 'i32'], returns: 'i32' },
108
+ proc_pid_rusage: { args: ['i32', 'i32', 'ptr'], returns: 'i32' },
109
+ });
110
+ if (!isLibprocSymbols(symbols)) {
111
+ throw new Error(`${LIBPROC} opened without proc_listchildpids and proc_pid_rusage`);
112
+ }
113
+ const usage = new BigUint64Array(RUSAGE_INFO_V0_WORDS);
114
+ const usageAt = ffi.ptr(usage);
115
+ let pids = new Int32Array(256);
116
+ return {
117
+ children(pid) {
118
+ // A full buffer may have cut the list short: it is read again into one twice the size.
119
+ for (;;) {
120
+ const count = symbols.proc_listchildpids(pid, ffi.ptr(pids), pids.byteLength);
121
+ if (count < pids.length) return count <= 0 ? [] : Array.from(pids.subarray(0, count));
122
+ pids = new Int32Array(pids.length * 2);
123
+ }
124
+ },
125
+ footprint(pid) {
126
+ return symbols.proc_pid_rusage(pid, RUSAGE_INFO_V0, usageAt) === 0 ? Number(usage[PHYS_FOOTPRINT]) : 0;
127
+ },
128
+ };
129
+ }
130
+
131
+ /** What the tree rooted at `root` holds: each member's `ri_phys_footprint`, the tree walked by `proc_listchildpids`. */
132
+ async function treeResidentBytes(root: number): Promise<number> {
133
+ footprints ??= loadFootprints();
134
+ const proc = await footprints;
135
+ const seen = new Set<number>();
136
+ const pending = [root];
137
+ let total = 0;
138
+ for (let pid = pending.pop(); pid !== undefined; pid = pending.pop()) {
139
+ if (seen.has(pid)) continue;
140
+ seen.add(pid);
141
+ total += proc.footprint(pid);
142
+ pending.push(...proc.children(pid));
143
+ }
144
+ return total;
145
+ }
146
+
147
+ export const darwinMemory: HostMemory = { readRunMemory, peakBytes: (maxRss) => maxRss, treeResidentBytes };
@@ -0,0 +1,147 @@
1
+ import { readFile } from 'node:fs/promises';
2
+ import { posix } from 'node:path';
3
+ import { childPids } from '../../process-children.linux.js';
4
+ import type { HostMemory, RunMemory, RunMemoryError, RunMemoryReading } from './host-memory.js';
5
+
6
+ /**
7
+ * Linux's memory mechanisms. The run's memory is the least of the host's `MemTotal` and every finite `memory.max` of
8
+ * the cgroup v2 the run's process is in and of its ancestors, up to the root of the mounted hierarchy: the kernel holds
9
+ * a cgroup to its ancestors' limits too, so a job in cowshed's `background` class, whose own `memory.max` is `max`, has
10
+ * the class's three quarters of the host. A host with no cgroup v2 hierarchy, or a hierarchy that sets no limit, gives
11
+ * the run `MemTotal`.
12
+ */
13
+
14
+ const MEMINFO = '/proc/meminfo';
15
+ const CGROUP = '/proc/self/cgroup';
16
+ const MOUNTINFO = '/proc/self/mountinfo';
17
+
18
+ /** A file's text, or the errno code and message its read failed with. */
19
+ export type FileRead =
20
+ | { readonly ok: true; readonly text: string }
21
+ | { readonly ok: false; readonly code: string; readonly message: string };
22
+
23
+ const failed = (code: RunMemoryError['code'], subject: string, cause: string): RunMemoryReading => ({
24
+ ok: false,
25
+ error: { operation: 'read-run-memory', code, subject, cause },
26
+ });
27
+ const unreadable = (subject: string, read: Extract<FileRead, { ok: false }>): RunMemoryReading =>
28
+ failed('unreadable', subject, `${read.code}: ${read.message}`);
29
+
30
+ /** A mount point or root as mountinfo writes it, its space, tab, newline and backslash escaped in octal. */
31
+ const decodeOctal = (field: string): string =>
32
+ field.replace(/\\([0-7]{3})/g, (_, octal: string) => String.fromCharCode(Number.parseInt(octal, 8)));
33
+
34
+ /** The cgroup v2 mounts mountinfo lists: the cgroup each shows (`root`) and where (`point`). */
35
+ function cgroup2Mounts(mountinfo: string): { root: string; point: string }[] {
36
+ const mounts: { root: string; point: string }[] = [];
37
+ for (const line of mountinfo.split('\n')) {
38
+ const fields = line.split(' ');
39
+ const separator = fields.indexOf('-');
40
+ const [root, point] = [fields[3], fields[4]];
41
+ if (separator !== -1 && fields[separator + 1] === 'cgroup2' && root !== undefined && point !== undefined) {
42
+ mounts.push({ root: decodeOctal(root), point: decodeOctal(point) });
43
+ }
44
+ }
45
+ return mounts;
46
+ }
47
+
48
+ /** The run's memory, reading every file through `read`. */
49
+ export async function linuxRunMemory(read: (path: string) => Promise<FileRead>): Promise<RunMemoryReading> {
50
+ const meminfo = await read(MEMINFO);
51
+ if (!meminfo.ok) return unreadable(MEMINFO, meminfo);
52
+ const total = /^MemTotal:\s+(\d+) kB$/m.exec(meminfo.text)?.[1];
53
+ if (total === undefined) return failed('malformed', MEMINFO, 'it names no MemTotal in kB');
54
+ let least: RunMemory = { bytes: Number(total) * 1024, source: `${MEMINFO} MemTotal` };
55
+ const cgroups = await read(CGROUP);
56
+ if (!cgroups.ok) return unreadable(CGROUP, cgroups);
57
+ const path = /^0::(\/.*)$/m.exec(cgroups.text)?.[1];
58
+ if (path === undefined) {
59
+ // No cgroup v2 hierarchy holds the run: the host's memory is its memory.
60
+ return { ok: true, memory: least };
61
+ }
62
+ const mounts = await read(MOUNTINFO);
63
+ if (!mounts.ok) return unreadable(MOUNTINFO, mounts);
64
+ const mount = cgroup2Mounts(mounts.text).find(
65
+ ({ root }) => root === '/' || path === root || path.startsWith(`${root}/`),
66
+ );
67
+ if (mount === undefined) {
68
+ return failed('unmounted', path, `${MOUNTINFO} lists no cgroup2 mount that shows the run's cgroup`);
69
+ }
70
+ const top = posix.resolve(mount.point);
71
+ const own = posix.resolve(top, `.${mount.root === '/' ? path : path.slice(mount.root.length)}`);
72
+ if (own !== top && !own.startsWith(`${top}/`)) {
73
+ return failed('unmounted', path, `the run's cgroup lies outside the cgroup2 mount at ${top}`);
74
+ }
75
+ for (let directory = own; ; ) {
76
+ const file = posix.join(directory, 'memory.max');
77
+ const limit = await read(file);
78
+ if (limit.ok) {
79
+ const value = limit.text.trim();
80
+ if (value !== 'max') {
81
+ if (!/^\d+$/.test(value)) {
82
+ return failed('malformed', file, `it holds ${JSON.stringify(value)}, not max or bytes`);
83
+ }
84
+ // A limit above the host's memory (a near-2^63 one included) is no constraint, and is never made a number.
85
+ if (BigInt(value) < BigInt(least.bytes)) least = { bytes: Number(value), source: file };
86
+ }
87
+ } else if (limit.code !== 'ENOENT') {
88
+ // ENOENT: the hierarchy's root, or a cgroup whose parent enables no memory controller, has no limit of its own.
89
+ return unreadable(file, limit);
90
+ }
91
+ if (directory === top) break;
92
+ directory = posix.dirname(directory);
93
+ }
94
+ return { ok: true, memory: least };
95
+ }
96
+
97
+ async function readText(path: string): Promise<FileRead> {
98
+ try {
99
+ return { ok: true, text: await readFile(path, 'utf8') };
100
+ } catch (error) {
101
+ const code = error instanceof Error && 'code' in error && typeof error.code === 'string' ? error.code : 'EIO';
102
+ return { ok: false, code, message: error instanceof Error ? error.message : String(error) };
103
+ }
104
+ }
105
+
106
+ /** Bun before 1.4.0 reports Linux's `ru_maxrss` as the kernel does, in KiB; from 1.4.0 it reports bytes. */
107
+ function peakBytes(maxRss: number, bunVersion: string): number {
108
+ const version = /^(\d+)\.(\d+)\./.exec(bunVersion);
109
+ if (version === null) {
110
+ throw new Error(`Bun names its version ${JSON.stringify(bunVersion)}, which is no semver`);
111
+ }
112
+ const [major, minor] = [Number(version[1]), Number(version[2])];
113
+ return major < 1 || (major === 1 && minor < 4) ? maxRss * 1024 : maxRss;
114
+ }
115
+
116
+ /** What the tree rooted at `root` holds resident: each member's `VmRSS`, the tree walked by the kernel's child links. */
117
+ async function treeResidentBytes(root: number): Promise<number> {
118
+ const seen = new Set<number>();
119
+ let level = [root];
120
+ let total = 0;
121
+ while (level.length > 0) {
122
+ const members = level.filter((pid) => !seen.has(pid));
123
+ for (const pid of members) seen.add(pid);
124
+ const [resident, children] = await Promise.all([Promise.all(members.map(residentBytes)), childPids(members)]);
125
+ total += resident.reduce((sum, bytes) => sum + bytes, 0);
126
+ level = children;
127
+ }
128
+ return total;
129
+ }
130
+
131
+ /**
132
+ * `VmRSS` from `/proc/<pid>/status`: what the process holds resident, anonymous, file-backed and shared memory together.
133
+ * A process that has exited (`ENOENT`, `ESRCH`) holds nothing, and so does a zombie, whose status names no `VmRSS`.
134
+ */
135
+ async function residentBytes(pid: number): Promise<number> {
136
+ let status: string;
137
+ try {
138
+ status = await readFile(`/proc/${pid}/status`, 'utf8');
139
+ } catch (error) {
140
+ if (error instanceof Error && 'code' in error && (error.code === 'ENOENT' || error.code === 'ESRCH')) return 0;
141
+ throw error;
142
+ }
143
+ const kib = /^VmRSS:\s+(\d+) kB$/m.exec(status)?.[1];
144
+ return kib === undefined ? 0 : Number(kib) * 1024;
145
+ }
146
+
147
+ export const linuxMemory: HostMemory = { readRunMemory: () => linuxRunMemory(readText), peakBytes, treeResidentBytes };