@smoothbricks/nx-plugin 0.7.5 → 0.7.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/README.md +20 -3
  2. package/dist/.build-inputs.json +12 -6
  3. package/dist/executors/bounded-exec/cpu-pool-thread.d.ts +3 -2
  4. package/dist/executors/bounded-exec/cpu-pool-thread.d.ts.map +1 -1
  5. package/dist/executors/bounded-exec/cpu-pool-thread.js +1 -1
  6. package/dist/executors/bounded-exec/cpu-pool.d.ts +24 -10
  7. package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -1
  8. package/dist/executors/bounded-exec/cpu-pool.js +96 -52
  9. package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
  10. package/dist/executors/bounded-exec/executor.js +115 -62
  11. package/dist/executors/bounded-exec/host-memory.d.ts +54 -0
  12. package/dist/executors/bounded-exec/host-memory.d.ts.map +1 -0
  13. package/dist/executors/bounded-exec/host-memory.darwin.d.ts +3 -0
  14. package/dist/executors/bounded-exec/host-memory.darwin.d.ts.map +1 -0
  15. package/dist/executors/bounded-exec/host-memory.darwin.js +115 -0
  16. package/dist/executors/bounded-exec/host-memory.js +29 -0
  17. package/dist/executors/bounded-exec/host-memory.linux.d.ts +14 -0
  18. package/dist/executors/bounded-exec/host-memory.linux.d.ts.map +1 -0
  19. package/dist/executors/bounded-exec/host-memory.linux.js +137 -0
  20. package/dist/executors/bounded-exec/leaf-peak.d.ts +56 -0
  21. package/dist/executors/bounded-exec/leaf-peak.d.ts.map +1 -0
  22. package/dist/executors/bounded-exec/leaf-peak.js +147 -0
  23. package/dist/executors/bounded-exec/mbx-reservations.d.ts +22 -0
  24. package/dist/executors/bounded-exec/mbx-reservations.d.ts.map +1 -1
  25. package/dist/executors/bounded-exec/mbx-reservations.js +22 -0
  26. package/dist/executors/bounded-exec/peak-waiter.d.ts +2 -0
  27. package/dist/executors/bounded-exec/peak-waiter.d.ts.map +1 -0
  28. package/dist/executors/bounded-exec/peak-waiter.js +79 -0
  29. package/dist/executors/bounded-exec/pool-admission.d.ts +38 -0
  30. package/dist/executors/bounded-exec/pool-admission.d.ts.map +1 -0
  31. package/dist/executors/bounded-exec/pool-admission.js +31 -0
  32. package/dist/executors/bounded-exec/run-cpu-pool.d.ts +6 -4
  33. package/dist/executors/bounded-exec/run-cpu-pool.d.ts.map +1 -1
  34. package/dist/executors/bounded-exec/run-cpu-pool.js +20 -7
  35. package/dist/executors/bounded-exec/verdict.d.ts +18 -0
  36. package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
  37. package/dist/executors/bounded-exec/verdict.js +21 -1
  38. package/managed/templates/github/actions/setup-devenv/action.yml +31 -2
  39. package/package.json +2 -2
  40. package/src/executors/bounded-exec/cpu-pool-thread.ts +4 -3
  41. package/src/executors/bounded-exec/cpu-pool.ts +111 -49
  42. package/src/executors/bounded-exec/executor.ts +121 -59
  43. package/src/executors/bounded-exec/host-memory.darwin.ts +147 -0
  44. package/src/executors/bounded-exec/host-memory.linux.ts +147 -0
  45. package/src/executors/bounded-exec/host-memory.ts +77 -0
  46. package/src/executors/bounded-exec/leaf-peak.ts +152 -0
  47. package/src/executors/bounded-exec/mbx-reservations.ts +30 -0
  48. package/src/executors/bounded-exec/peak-waiter.ts +115 -0
  49. package/src/executors/bounded-exec/pool-admission.ts +64 -0
  50. package/src/executors/bounded-exec/run-cpu-pool.ts +20 -6
  51. package/src/executors/bounded-exec/verdict.ts +30 -1
  52. package/test-timings.json +136 -132
@@ -1,7 +1,7 @@
1
1
  import { spawn } from 'node:child_process';
2
2
  import { accessSync, constants } from 'node:fs';
3
3
  import { mkdtemp, rm } from 'node:fs/promises';
4
- import { availableParallelism, userInfo } from 'node:os';
4
+ import { availableParallelism, constants as osConstants, userInfo } from 'node:os';
5
5
  import { delimiter, isAbsolute, join, resolve as resolvePath } from 'node:path';
6
6
  import { createInterface } from 'node:readline';
7
7
  import { Duplex, Readable } from 'node:stream';
@@ -11,9 +11,10 @@ import { admissionWaitMs, earliestBound, NO_ADMISSION_WAIT, stepAdmissionWait, s
11
11
  import { CPU_POOL_ENV, createCpuPool, readInheritedCpuPool, takeParentCpuTokens, } from './cpu-pool.js';
12
12
  import { GATEWAY_SOCKET } from './gateway-lease.js';
13
13
  import { captureHangDiagnostics, HANG_CAPTURE_MS } from './hang-diagnostics.js';
14
+ import { PEAK_FD, peakWaiter, readWaiterReport, underWaiter } from './leaf-peak.js';
14
15
  import { askedCores, MBX_GRANT_FD, mbxIdleGated, mbxLeaf, mbxPool, withCountedReservation, withMbxLeaf, } from './mbx-pool.js';
15
16
  import { MBX_QUEUE_POLL_MS, snapshotMbxQueue } from './mbx-queue.js';
16
- import { RESERVATIONS_ENV, reserveCores } from './mbx-reservations.js';
17
+ import { describeUnserved, RESERVATIONS_ENV, reserveCores, runEndpoint } from './mbx-reservations.js';
17
18
  import { createNoSpaceWatch } from './no-space.js';
18
19
  import { COWSHED_EXECUTABLE_ENV, cowshedSubtreeOwner, HANDLE_FD, readAdmission, SUPERVISOR_SOCKET_ENV, } from './owned-subtree.js';
19
20
  import { createRunProgress, withProgressReport, withTestResultReport } from './progress.js';
@@ -22,14 +23,9 @@ import { createRunnerLifecycle, withFixtureLifecycle, withRunnerLifecycle } from
22
23
  import { CPU_TOKENS_ENV, runnerDemand, sizeRunner, sizeRunnerToGrant } from './runner-sizing.js';
23
24
  import { checkRuntimePin, describeRuntimePinRefusal } from './runtime-pin.js';
24
25
  import { createTestResultWriter } from './test-result-writer.js';
25
- import { BUN_TEST, clearTaskDirectory, describeAdmissionQueue, describeNoSpace, JUNIT_FILE, JUNIT_REPORT_ENV, judgeOperationalFailure, judgeRun, readJunitReport, taskDirectory, withJunitReport, writeRecord, } from './verdict.js';
26
+ import { BUN_TEST, clearTaskDirectory, describeAdmissionQueue, describeNoSpace, JUNIT_FILE, JUNIT_REPORT_ENV, judgeOperationalFailure, judgeRun, readJunitReport, readLastPeak, taskDirectory, withJunitReport, writeRecord, } from './verdict.js';
26
27
  const DEFAULT_KILL_AFTER_MS = 10_000;
27
28
  const REAP_AFTER_FORCE_KILL_MS = 2_000;
28
- const EXIT_CODE_BY_SIGNAL = {
29
- SIGHUP: 129,
30
- SIGINT: 130,
31
- SIGTERM: 143,
32
- };
33
29
  /** The pool descriptor: only a grant may give it, so neither the target's nor the inherited environment passes it on. */
34
30
  const POOL_ENV = [CPU_TOKENS_ENV, CPU_POOL_ENV];
35
31
  /**
@@ -148,6 +144,8 @@ export function recordContext(context) {
148
144
  export async function runBoundedExec(options, context, killer, tempVolume, inherited = null, captureDiagnostics = captureHangDiagnostics, owner = null, queueHost) {
149
145
  const cwd = resolveCwd(options.cwd, context.root);
150
146
  const directory = context.record?.directory ?? null;
147
+ // The task's last recorded peak is its charge in its parent's pool, read before the clear below removes it.
148
+ const lastPeak = directory !== null && inherited !== null ? await readLastPeak(directory) : null;
151
149
  // A previous run's verdict and report go first, so neither can be read as this run's.
152
150
  if (directory !== null) {
153
151
  await clearTaskDirectory(directory);
@@ -196,17 +194,26 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
196
194
  // Nobody is left to tell.
197
195
  }
198
196
  };
197
+ // A run refused before its command exists says why and records an operational failure, and no peak: nothing ran.
198
+ const refuseUnstarted = async (message) => {
199
+ appendStderr(message);
200
+ if (context.record !== undefined) {
201
+ const { task, directory: recordDirectory } = context.record;
202
+ await writeRecord(recordDirectory, {
203
+ task: task.id,
204
+ hash: task.hash,
205
+ verdict: judgeOperationalFailure(1, null),
206
+ peakBytes: null,
207
+ });
208
+ }
209
+ return { success: false, terminalOutput: outputChunks.join('') };
210
+ };
199
211
  // The fixture pair names one segment: a fixture with no build budget, or a budget with no fixture, is a
200
212
  // target that does not say what it runs, so it does not run. Units are counted at the end of that segment.
201
213
  if ((options.fixtureCommand === undefined) !== (options.fixtureTimeoutMs === undefined) ||
202
214
  (options.unitsCommand !== undefined && options.fixtureCommand === undefined) ||
203
215
  (fixture !== null && !(fixture.timeoutMs > 0))) {
204
- appendStderr('bounded-exec: fixtureCommand and a positive fixtureTimeoutMs must be named together, and unitsCommand only with them; refusing to run the command\n');
205
- if (context.record !== undefined) {
206
- const { task, directory: recordDirectory } = context.record;
207
- await writeRecord(recordDirectory, { task: task.id, hash: task.hash, verdict: judgeOperationalFailure(1, null) });
208
- }
209
- return { success: false, terminalOutput: outputChunks.join('') };
216
+ return refuseUnstarted('bounded-exec: fixtureCommand and a positive fixtureTimeoutMs must be named together, and unitsCommand only with them; refusing to run the command\n');
210
217
  }
211
218
  // A leg that runs bun runs the Bun its workspace declares, or it does not run. This comes before the RAM lease and the
212
219
  // CPU tokens: a refusal that waited in its pool's queue first would say nothing for minutes.
@@ -218,13 +225,13 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
218
225
  pathSource: options.env?.PATH === undefined ? 'the inherited environment' : "the target's env",
219
226
  });
220
227
  if (runtime !== null) {
221
- appendStderr(describeRuntimePinRefusal(runtime, context.record?.task.id));
222
- if (context.record !== undefined) {
223
- const { task, directory: recordDirectory } = context.record;
224
- await writeRecord(recordDirectory, { task: task.id, hash: task.hash, verdict: judgeOperationalFailure(1, null) });
225
- }
226
- return { success: false, terminalOutput: outputChunks.join('') };
228
+ return refuseUnstarted(describeRuntimePinRefusal(runtime, context.record?.task.id));
229
+ }
230
+ if (lastPeak !== null && !lastPeak.ok) {
231
+ return refuseUnstarted(`bounded-exec: ${lastPeak.error}; it is this task's memory charge, so the command does not run\n`);
227
232
  }
233
+ // A task with no recorded peak is charged nothing (null) until it has one.
234
+ const charge = lastPeak?.ok === true ? lastPeak.peakBytes : null;
228
235
  let lease = null;
229
236
  let ordinaryTemp = null;
230
237
  // Before spawn there is nothing to retire. Once started, only the owner's terminal retirement permits deletion.
@@ -364,7 +371,7 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
364
371
  let parentRefusal = null;
365
372
  let parentWait;
366
373
  if (inherited !== null) {
367
- const fromParent = await takeParentCpuTokens(inherited, demand.want);
374
+ const fromParent = await takeParentCpuTokens(inherited, demand.want, charge);
368
375
  if (fromParent.granted) {
369
376
  grant = fromParent;
370
377
  }
@@ -386,7 +393,10 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
386
393
  if (grant !== null) {
387
394
  sized = sizeRunner(demand, requested, grant.tokens);
388
395
  cpu = { tokens: grant.tokens, want: demand.want, waitedMs };
389
- appendStderr(`cowshed: cpu-pool wait done elapsed=${waitedMs}ms status=ok tokens=${grant.tokens}/${demand.want} runner=${demand.kind}\n`);
396
+ appendStderr(`cowshed: cpu-pool wait done elapsed=${waitedMs}ms status=ok tokens=${grant.tokens}/${demand.want} runner=${demand.kind} memory=${charge ?? 'none'}\n`);
397
+ if (grant.oversized !== null) {
398
+ appendStderr(`cowshed: cpu-pool granted while nothing else holds memory: this task's last peak of ${charge} bytes alone exceeds the run's ${grant.oversized.memory} bytes\n`);
399
+ }
390
400
  }
391
401
  else if (leaf?.kind === 'reserve') {
392
402
  sized = { command: sizeRunnerToGrant(leaf.demand, requested), env: {} };
@@ -444,7 +454,7 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
444
454
  };
445
455
  // Records the run's verdict when it has a task to record as. An `expiry` is set only when a bound ended the run.
446
456
  // Observation and ownership failures remain operational even after valid queue evidence; cancellation alone does not.
447
- const recordRun = async (exitCode, expiry, elapsedMs, operational, noSpace, lockWaits) => {
457
+ const recordRun = async (exitCode, expiry, elapsedMs, operational, noSpace, lockWaits, peakBytes) => {
448
458
  if (context.record === undefined) {
449
459
  return;
450
460
  }
@@ -485,6 +495,7 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
485
495
  verdict,
486
496
  ...(cpu ? { cpu } : {}),
487
497
  ...(parentWait ? { cpuWait: parentWait } : {}),
498
+ peakBytes,
488
499
  });
489
500
  };
490
501
  // A run refused before its command starts says why, gives back its grant and RAM, and records the failure.
@@ -493,22 +504,24 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
493
504
  await releaseCpu();
494
505
  // The command never started, so its lease holds nothing of its own to keep.
495
506
  await releaseLease(false);
496
- await recordRun(1, null, Date.now() - asked, false, null, []);
507
+ await recordRun(1, null, Date.now() - asked, false, null, [], null);
497
508
  return { success: false, terminalOutput: outputChunks.join('') };
498
509
  };
499
510
  if (parentRefusal !== null) {
500
511
  return refuse(`cowshed: cpu-pool parent pool refused: ${parentRefusal}; refusing to run the command\n`);
501
512
  }
502
513
  // A reserved leaf takes its cores from its Nx run's mbx reservation endpoint (mbx-reservations.ts). A run that serves
503
- // none said why in its own output; the leaf does not run beside the pool.
504
- const reserved = leaf?.kind === 'reserve' ? { leaf, endpoint: process.env[RESERVATIONS_ENV] ?? '' } : null;
505
- if (reserved !== null && reserved.endpoint === '') {
506
- return refuse(`cowshed: a reserved leaf takes its cores from its Nx run's mbx reservation endpoint, and ${RESERVATIONS_ENV} names none (the run's preTasksExecution serves it); refusing to run the command\n`);
514
+ // none said why in its own output, which a gate's failures-only output never shows, so the refusal says it too; the
515
+ // leaf does not run beside the pool.
516
+ const endpoint = leaf?.kind === 'reserve' ? runEndpoint(process.env) : null;
517
+ if (endpoint !== null && endpoint.kind !== 'served') {
518
+ return refuse(`cowshed: a reserved leaf takes its cores from its Nx run's mbx reservation endpoint, and ${RESERVATIONS_ENV} names none: ${describeUnserved(endpoint)}; refusing to run the command\n`);
507
519
  }
520
+ const reserved = leaf?.kind === 'reserve' && endpoint?.kind === 'served' ? { leaf, endpoint: endpoint.socket } : null;
508
521
  // A granted run serves a sub-pool of its own grant, so its descendants draw from that and never from above it.
509
522
  // A pool that cannot be set up refuses the run: no granted command runs beside its grant.
510
523
  if (grant !== null) {
511
- const created = await createCpuPool(grant.tokens, grant.closed);
524
+ const created = await createCpuPool(grant.tokens, grant.closed, null);
512
525
  if (!created.ok) {
513
526
  return refuse(`cowshed: cpu pool unavailable: ${created.error}; refusing to run the command\n`);
514
527
  }
@@ -555,6 +568,17 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
555
568
  const grantSlots = leaf !== null ? ['ignore', 'pipe'] : [];
556
569
  const rootGrantSlot = leaf !== null ? ['pipe'] : [];
557
570
  const root = owner?.rootArgv(shellCommand);
571
+ // The command's own descriptors past stderr, from fd 3: the lifecycle channel, the subtree admission, mbx's grant.
572
+ const descriptors = root === undefined
573
+ ? hooked === null && leaf === null
574
+ ? []
575
+ : [hooked === null ? 'ignore' : 'pipe', ...grantSlots]
576
+ : [hooked === null ? 'ignore' : 'pipe', 'pipe', ...rootGrantSlot];
577
+ // A run that records its verdict records its command's peak too: the command runs under a waiter that reports it.
578
+ const waiter = context.record === undefined ? null : peakWaiter();
579
+ const measured = waiter?.kind === 'waiter'
580
+ ? underWaiter(waiter.runtime, root?.file ?? '/bin/sh', root?.args ?? ['-c', shellCommand], descriptors)
581
+ : null;
558
582
  let child;
559
583
  const results = directory === null ? null : await createTestResultWriter(directory, { cwd, root: context.root });
560
584
  if (results !== null && !results.ok)
@@ -562,32 +586,48 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
562
586
  resultWriter = results?.ok ? results.writer : null;
563
587
  try {
564
588
  // fd 3 is the runner lifecycle's duplex socket when the command is hooked; the subtree admission takes fd 4, which
565
- // cowshed closes fd 4 before exec; fd 5 is a Cargo leaf's mbx admission stream or a reserved leaf's ask and grant.
589
+ // cowshed closes fd 4 before exec; fd 5 is a Cargo leaf's mbx admission stream or a reserved leaf's ask and grant;
590
+ // the waiter, when there is one, passes them on and reports the command's peak on fd 6.
566
591
  child =
567
- root === undefined
568
- ? spawn(shellCommand, [], {
592
+ measured !== null
593
+ ? spawn(measured.file, measured.args, {
569
594
  cwd,
570
595
  env: mergeEnv(lease?.directory ?? ordinaryTemp, env),
571
- shell: true,
572
- stdio: hooked === null && leaf === null
573
- ? 'pipe'
574
- : ['pipe', 'pipe', 'pipe', hooked === null ? 'ignore' : 'pipe', ...grantSlots],
596
+ stdio: measured.stdio,
575
597
  detached: process.platform !== 'win32',
576
598
  windowsHide: true,
577
599
  })
578
- : spawn(root.file, root.args, {
579
- cwd,
580
- env: mergeEnv(lease?.directory ?? ordinaryTemp, env),
581
- stdio: ['pipe', 'pipe', 'pipe', hooked === null ? 'ignore' : 'pipe', 'pipe', ...rootGrantSlot],
582
- detached: process.platform !== 'win32',
583
- windowsHide: true,
584
- });
600
+ : root === undefined
601
+ ? spawn(shellCommand, [], {
602
+ cwd,
603
+ env: mergeEnv(lease?.directory ?? ordinaryTemp, env),
604
+ shell: true,
605
+ stdio: descriptors.length === 0 ? 'pipe' : ['pipe', 'pipe', 'pipe', ...descriptors],
606
+ detached: process.platform !== 'win32',
607
+ windowsHide: true,
608
+ })
609
+ : spawn(root.file, root.args, {
610
+ cwd,
611
+ env: mergeEnv(lease?.directory ?? ordinaryTemp, env),
612
+ stdio: ['pipe', 'pipe', 'pipe', ...descriptors],
613
+ detached: process.platform !== 'win32',
614
+ windowsHide: true,
615
+ });
585
616
  }
586
617
  catch (error) {
587
618
  // Nothing started, so nothing runs under the grant: it goes back here, with the RAM lease.
588
619
  return refuse(`cowshed: could not start the command: ${error instanceof Error ? error.message : String(error)}\n`);
589
620
  }
590
621
  subtreeRetired = false;
622
+ const reportChannel = child.stdio.at(PEAK_FD);
623
+ const waiterReport = measured === null ? null : readWaiterReport(reportChannel instanceof Readable ? reportChannel : null);
624
+ // The leaf's runner is its command: under a waiter, which leads the group, the child the waiter names; else the
625
+ // group's leader itself. A hang snapshot names and samples it (specs/cowshed/08_testing.md).
626
+ const runnerPid = waiterReport?.started ?? Promise.resolve(child.pid ?? null);
627
+ let runner = null;
628
+ void runnerPid.then((pid) => {
629
+ runner = pid;
630
+ });
591
631
  if (root !== undefined) {
592
632
  const channel = child.stdio[HANDLE_FD];
593
633
  const leased = lease;
@@ -643,20 +683,24 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
643
683
  captureTimer = setTimeout(() => {
644
684
  if (state.settled || state.expiry !== null || idleCapture !== capture)
645
685
  return;
646
- const pid = child.pid;
647
- if (pid === undefined) {
686
+ const group = child.pid;
687
+ if (group === undefined) {
648
688
  idleCapture.result = { kind: 'failed', detail: 'runner PID unavailable; no holder conclusion' };
649
689
  return;
650
690
  }
651
691
  idleCapture.result = { kind: 'pending', startedAtMs: Date.now() };
652
- void Promise.resolve()
653
- .then(() => captureDiagnostics({
654
- pid,
655
- pgid: process.platform === 'win32' ? null : pid,
656
- expiresAtMs: capture.expiresAtMs,
657
- deadlineMs: Math.max(0, Math.min(HANG_CAPTURE_MS, capture.expiresAtMs - Date.now())),
658
- signal: capture.controller.signal,
659
- }))
692
+ void runnerPid
693
+ .then((pid) => {
694
+ if (pid === null)
695
+ throw new Error('runner PID unavailable: its waiter named no command; no holder conclusion');
696
+ return captureDiagnostics({
697
+ pid,
698
+ pgid: process.platform === 'win32' ? null : group,
699
+ expiresAtMs: capture.expiresAtMs,
700
+ deadlineMs: Math.max(0, Math.min(HANG_CAPTURE_MS, capture.expiresAtMs - Date.now())),
701
+ signal: capture.controller.signal,
702
+ });
703
+ })
660
704
  .then((snapshot) => {
661
705
  if (idleCapture === capture)
662
706
  idleCapture.result = { kind: 'captured', snapshot };
@@ -900,7 +944,7 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
900
944
  .finally(releaseCpu)
901
945
  .finally(() => releaseLease(true))
902
946
  .finally(releaseReservation)
903
- .finally(async () => recordRun(signalToExitCode(signal), state.expiry, Date.now() - startedAt, state.poolCloseError !== null || lifecycleFailure !== null || state.retireError !== null, state.noSpace === null ? null : await state.noSpace, state.lockWaits))
947
+ .finally(async () => recordRun(signalToExitCode(signal), state.expiry, Date.now() - startedAt, state.poolCloseError !== null || lifecycleFailure !== null || state.retireError !== null, state.noSpace === null ? null : await state.noSpace, state.lockWaits, null))
904
948
  .finally(() => process.kill(process.pid, signal));
905
949
  };
906
950
  const onSigint = () => onTerminationSignal('SIGINT');
@@ -957,7 +1001,7 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
957
1001
  if (bound.kind === 'idle') {
958
1002
  const observed = idleCapture?.result;
959
1003
  state.lockWaits = observed?.kind === 'captured' ? observed.snapshot.lockWaits : [];
960
- appendStderr(`Hang diagnostics: runner pid=${child.pid ?? 'unavailable'} pgid=${process.platform === 'win32' ? 'unavailable' : (child.pid ?? 'unavailable')} expiryAtMs=${Date.now()} scheduledExpiryAtMs=${idleCapture?.expiresAtMs ?? 'unavailable'}\n`);
1004
+ appendStderr(`Hang diagnostics: runner pid=${runner ?? 'unavailable'} pgid=${process.platform === 'win32' ? 'unavailable' : (child.pid ?? 'unavailable')} expiryAtMs=${Date.now()} scheduledExpiryAtMs=${idleCapture?.expiresAtMs ?? 'unavailable'}\n`);
961
1005
  if (observed?.kind === 'captured') {
962
1006
  appendStderr(`Hang diagnostics: outcome=${observed.snapshot.outcome} capturedAtMs=${observed.snapshot.capturedAtMs} actualExpiryOffsetMs=${Date.now() - observed.snapshot.capturedAtMs}\n${observed.snapshot.text}`);
963
1007
  }
@@ -1155,9 +1199,14 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
1155
1199
  await releaseCpu();
1156
1200
  const code = exitCode ?? signalToExitCode(exitSignal);
1157
1201
  if (code !== 0 && state.expiry === null) {
1202
+ // No bound of this executor ended it, so a signal it died of came from outside (the kernel's OOM killer sends
1203
+ // SIGKILL): the signal is named, never folded into a bare status.
1204
+ const ended = exitCode === null && exitSignal !== null
1205
+ ? `was killed by ${exitSignal} (status ${code})`
1206
+ : `exited with status ${code}`;
1158
1207
  appendStderr(fixture !== null && !mainStarted
1159
- ? `Fixture command exited with status ${code}: ${segment ?? fixture.command}\n`
1160
- : `Command exited with status ${code}: ${command}\n`);
1208
+ ? `Fixture command ${ended}: ${segment ?? fixture.command}\n`
1209
+ : `Command ${ended}: ${command}\n`);
1161
1210
  }
1162
1211
  if (state.expiry !== null && !state.forceKillNeeded) {
1163
1212
  appendStderr(`Timed out command exited after graceful termination: ${command}\n`);
@@ -1177,10 +1226,16 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
1177
1226
  }
1178
1227
  await releaseLease(code !== 0 || state.expiry !== null || state.poolCloseError !== null || lifecycleFailure !== null);
1179
1228
  releaseReservation();
1229
+ // The command's peak, as its waiter reported it once both ended; a peak that was not measured is said, never zero.
1230
+ const reported = waiterReport === null ? (waiter?.kind === 'unmeasured' ? waiter : null) : await waiterReport.peak;
1231
+ if (reported?.kind === 'unmeasured') {
1232
+ appendStderr(`bounded-exec: this run's memory peak was not measured (${reported.reason}); its record names none\n`);
1233
+ }
1234
+ const peakBytes = reported?.kind === 'measured' ? reported.bytes : null;
1180
1235
  // Broken resource ownership, a lost native lifecycle, or leftovers that could not be retired fail even if a shell
1181
1236
  // branch later exited 0: its verdict is an ordinary operational failure, never a successful or silently unbounded run.
1182
1237
  const operationalFailure = state.poolCloseError !== null || lifecycleFailure !== null || state.retireError !== null;
1183
- await recordRun(code, state.expiry, Date.now() - startedAt, operationalFailure, seen, state.lockWaits);
1238
+ await recordRun(code, state.expiry, Date.now() - startedAt, operationalFailure, seen, state.lockWaits, peakBytes);
1184
1239
  return {
1185
1240
  success: !operationalFailure && resultFailure === null && state.expiry === null && code === 0,
1186
1241
  terminalOutput: outputChunks.join(''),
@@ -1239,11 +1294,9 @@ function targetEnv(env, say) {
1239
1294
  }
1240
1295
  return kept;
1241
1296
  }
1297
+ /** The status a shell gives a command a signal ended: 128 plus the signal's number (SIGKILL 137, SIGTERM 143). */
1242
1298
  function signalToExitCode(signal) {
1243
- if (!signal) {
1244
- return 1;
1245
- }
1246
- return EXIT_CODE_BY_SIGNAL[signal] ?? 1;
1299
+ return signal === null ? 1 : 128 + osConstants.signals[signal];
1247
1300
  }
1248
1301
  /** Resolves when `promise` settles or `ms` passes, whichever is first, leaving no timer behind. */
1249
1302
  async function settledWithin(promise, ms) {
@@ -0,0 +1,54 @@
1
+ /**
2
+ * The memory facts a run's CPU pool charges by (specs/cowshed/05_gateway.md, "Without mbx, every Nx run budgets
3
+ * itself"; 13_telemetry.md, "Job resource peaks"), read by each platform's own mechanism: `host-memory.linux.ts`,
4
+ * `host-memory.darwin.ts`. Callers read them here and never ask which platform they run on.
5
+ */
6
+ export interface HostMemory {
7
+ /** The memory the run's pool charges its leaves' peaks against. */
8
+ readRunMemory(): Promise<RunMemoryReading>;
9
+ /**
10
+ * A leaf's peak in bytes, from the `maxRSS` that Bun's `Subprocess.resourceUsage()` on `bunVersion` reports for
11
+ * it: the exited leaf's `ru_maxrss`.
12
+ */
13
+ peakBytes(maxRss: number, bunVersion: string): number;
14
+ /**
15
+ * What the process tree rooted at `root` holds resident at this instant: the sum over `root` and every descendant the
16
+ * kernel links to it, each process's footprint where the platform keeps one (macOS `ri_phys_footprint`, which counts
17
+ * its compressed pages) and its resident set where it does not (Linux `VmRSS`). A process that exits between the
18
+ * listing and its read adds nothing, and a root that has exited holds nothing.
19
+ */
20
+ treeResidentBytes(root: number): Promise<number>;
21
+ }
22
+ /** The run's memory in bytes, and where it was read: the file or sysctl that holds it. */
23
+ export interface RunMemory {
24
+ readonly bytes: number;
25
+ readonly source: string;
26
+ }
27
+ /** Why the run's memory could not be read: the file or sysctl, and what reading it gave. */
28
+ export interface RunMemoryError {
29
+ readonly operation: 'read-run-memory';
30
+ /**
31
+ * `unreadable`: reading it failed; `malformed`: it held no byte count; `unmounted`: the run's cgroup v2 hierarchy is
32
+ * mounted nowhere this process can see; `unsupported-platform`: the platform has no reader.
33
+ */
34
+ readonly code: 'unreadable' | 'malformed' | 'unmounted' | 'unsupported-platform';
35
+ readonly subject: string;
36
+ readonly cause: string;
37
+ }
38
+ export type RunMemoryReading = {
39
+ readonly ok: true;
40
+ readonly memory: RunMemory;
41
+ } | {
42
+ readonly ok: false;
43
+ readonly error: RunMemoryError;
44
+ };
45
+ /** A run whose memory cannot be read does not start: it is not shown to have room. */
46
+ export declare class RunMemoryUnreadable extends Error {
47
+ readonly error: RunMemoryError;
48
+ constructor(error: RunMemoryError);
49
+ }
50
+ /** This platform's mechanisms, or null where it has none: there a leaf's peak goes unmeasured. */
51
+ export declare const HOST_MEMORY: HostMemory | null;
52
+ /** The run's memory on this platform, or why it cannot be read. */
53
+ export declare function readRunMemory(): Promise<RunMemoryReading>;
54
+ //# sourceMappingURL=host-memory.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"host-memory.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/host-memory.ts"],"names":[],"mappings":"AAGA;;;;GAIG;AACH,MAAM,WAAW,UAAU;IACzB,mEAAmE;IACnE,aAAa,IAAI,OAAO,CAAC,gBAAgB,CAAC,CAAC;IAC3C;;;OAGG;IACH,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,MAAM,CAAC;IACtD;;;;;OAKG;IACH,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CAClD;AAED,0FAA0F;AAC1F,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED,4FAA4F;AAC5F,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,SAAS,EAAE,iBAAiB,CAAC;IACtC;;;OAGG;IACH,QAAQ,CAAC,IAAI,EAAE,YAAY,GAAG,WAAW,GAAG,WAAW,GAAG,sBAAsB,CAAC;IACjF,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,MAAM,gBAAgB,GACxB;IAAE,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAA;CAAE,GACjD;IAAE,QAAQ,CAAC,EAAE,EAAE,KAAK,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,cAAc,CAAA;CAAE,CAAC;AAE3D,sFAAsF;AACtF,qBAAa,mBAAoB,SAAQ,KAAK;IAChC,QAAQ,CAAC,KAAK,EAAE,cAAc;IAA1C,YAAqB,KAAK,EAAE,cAAc,EAKzC;CACF;AAID,kGAAkG;AAClG,eAAO,MAAM,WAAW,EAAE,UAAU,GAAG,IAA0C,CAAC;AAElF,mEAAmE;AACnE,wBAAgB,aAAa,IAAI,OAAO,CAAC,gBAAgB,CAAC,CAazD"}
@@ -0,0 +1,3 @@
1
+ import type { HostMemory } from './host-memory.js';
2
+ export declare const darwinMemory: HostMemory;
3
+ //# sourceMappingURL=host-memory.darwin.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"host-memory.darwin.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/host-memory.darwin.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,UAAU,EAAoB,MAAM,kBAAkB,CAAC;AAgJrE,eAAO,MAAM,YAAY,EAAE,UAAgF,CAAC"}
@@ -0,0 +1,115 @@
1
+ import { execFile } from 'node:child_process';
2
+ import { promisify } from 'node:util';
3
+ /**
4
+ * macOS's memory mechanisms. A run's memory is the host's, `hw.memsize`: macOS confines no job to a memory limit of
5
+ * its own. Bun reports a leaf's `ru_maxrss` in bytes, as macOS does.
6
+ */
7
+ const run = promisify(execFile);
8
+ const HW_MEMSIZE = 'hw.memsize';
9
+ async function readRunMemory() {
10
+ let stdout;
11
+ try {
12
+ ({ stdout } = await run('sysctl', ['-n', HW_MEMSIZE]));
13
+ }
14
+ catch (error) {
15
+ return {
16
+ ok: false,
17
+ error: {
18
+ operation: 'read-run-memory',
19
+ code: 'unreadable',
20
+ subject: HW_MEMSIZE,
21
+ cause: error instanceof Error ? error.message : String(error),
22
+ },
23
+ };
24
+ }
25
+ const value = stdout.trim();
26
+ const bytes = Number(value);
27
+ if (!/^\d+$/.test(value) || !Number.isSafeInteger(bytes) || bytes === 0) {
28
+ return {
29
+ ok: false,
30
+ error: {
31
+ operation: 'read-run-memory',
32
+ code: 'malformed',
33
+ subject: HW_MEMSIZE,
34
+ cause: `sysctl printed ${JSON.stringify(value)}, not a byte count`,
35
+ },
36
+ };
37
+ }
38
+ return { ok: true, memory: { bytes, source: HW_MEMSIZE } };
39
+ }
40
+ /**
41
+ * Bun's FFI, as the peak waiter reaches it. Node, which runs the executor, loads this module too and has no `bun:ffi`;
42
+ * only the waiter, on Bun, reads a tree's footprint, so the FFI loads on its first read, by a specifier the compiler
43
+ * does not resolve against Node's types.
44
+ */
45
+ const BUN_FFI = 'bun:ffi';
46
+ const LIBPROC = '/usr/lib/libproc.dylib';
47
+ /** `proc_pid_rusage`'s oldest flavor, whose fields every later one starts with. */
48
+ const RUSAGE_INFO_V0 = 0;
49
+ /** `rusage_info_v0`: a 16-byte uuid, then ten u64 counters; `ri_phys_footprint` is the eighth, u64 index 9. */
50
+ const RUSAGE_INFO_V0_WORDS = 12;
51
+ const PHYS_FOOTPRINT = 9;
52
+ function isBunFfi(module) {
53
+ return (typeof module === 'object' &&
54
+ module !== null &&
55
+ 'dlopen' in module &&
56
+ typeof module.dlopen === 'function' &&
57
+ 'ptr' in module &&
58
+ typeof module.ptr === 'function');
59
+ }
60
+ function isLibprocSymbols(symbols) {
61
+ return (typeof symbols === 'object' &&
62
+ symbols !== null &&
63
+ 'proc_listchildpids' in symbols &&
64
+ typeof symbols.proc_listchildpids === 'function' &&
65
+ 'proc_pid_rusage' in symbols &&
66
+ typeof symbols.proc_pid_rusage === 'function');
67
+ }
68
+ let footprints;
69
+ async function loadFootprints() {
70
+ const ffi = await import(BUN_FFI);
71
+ if (!isBunFfi(ffi)) {
72
+ throw new Error(`${BUN_FFI} names no dlopen and ptr: a tree's footprint is read on Bun, by the peak waiter`);
73
+ }
74
+ const { symbols } = ffi.dlopen(LIBPROC, {
75
+ proc_listchildpids: { args: ['i32', 'ptr', 'i32'], returns: 'i32' },
76
+ proc_pid_rusage: { args: ['i32', 'i32', 'ptr'], returns: 'i32' },
77
+ });
78
+ if (!isLibprocSymbols(symbols)) {
79
+ throw new Error(`${LIBPROC} opened without proc_listchildpids and proc_pid_rusage`);
80
+ }
81
+ const usage = new BigUint64Array(RUSAGE_INFO_V0_WORDS);
82
+ const usageAt = ffi.ptr(usage);
83
+ let pids = new Int32Array(256);
84
+ return {
85
+ children(pid) {
86
+ // A full buffer may have cut the list short: it is read again into one twice the size.
87
+ for (;;) {
88
+ const count = symbols.proc_listchildpids(pid, ffi.ptr(pids), pids.byteLength);
89
+ if (count < pids.length)
90
+ return count <= 0 ? [] : Array.from(pids.subarray(0, count));
91
+ pids = new Int32Array(pids.length * 2);
92
+ }
93
+ },
94
+ footprint(pid) {
95
+ return symbols.proc_pid_rusage(pid, RUSAGE_INFO_V0, usageAt) === 0 ? Number(usage[PHYS_FOOTPRINT]) : 0;
96
+ },
97
+ };
98
+ }
99
+ /** What the tree rooted at `root` holds: each member's `ri_phys_footprint`, the tree walked by `proc_listchildpids`. */
100
+ async function treeResidentBytes(root) {
101
+ footprints ??= loadFootprints();
102
+ const proc = await footprints;
103
+ const seen = new Set();
104
+ const pending = [root];
105
+ let total = 0;
106
+ for (let pid = pending.pop(); pid !== undefined; pid = pending.pop()) {
107
+ if (seen.has(pid))
108
+ continue;
109
+ seen.add(pid);
110
+ total += proc.footprint(pid);
111
+ pending.push(...proc.children(pid));
112
+ }
113
+ return total;
114
+ }
115
+ export const darwinMemory = { readRunMemory, peakBytes: (maxRss) => maxRss, treeResidentBytes };
@@ -0,0 +1,29 @@
1
+ import { darwinMemory } from './host-memory.darwin.js';
2
+ import { linuxMemory } from './host-memory.linux.js';
3
+ /** A run whose memory cannot be read does not start: it is not shown to have room. */
4
+ export class RunMemoryUnreadable extends Error {
5
+ error;
6
+ constructor(error) {
7
+ super(`bounded-exec: this run's memory cannot be read (${error.operation} ${error.code} ${error.subject}: ${error.cause}); a run whose memory cannot be read is not shown to have room, so it does not start`);
8
+ this.error = error;
9
+ this.name = 'RunMemoryUnreadable';
10
+ }
11
+ }
12
+ const PLATFORMS = { darwin: darwinMemory, linux: linuxMemory };
13
+ /** This platform's mechanisms, or null where it has none: there a leaf's peak goes unmeasured. */
14
+ export const HOST_MEMORY = PLATFORMS[process.platform] ?? null;
15
+ /** The run's memory on this platform, or why it cannot be read. */
16
+ export function readRunMemory() {
17
+ if (HOST_MEMORY === null) {
18
+ return Promise.resolve({
19
+ ok: false,
20
+ error: {
21
+ operation: 'read-run-memory',
22
+ code: 'unsupported-platform',
23
+ subject: process.platform,
24
+ cause: 'only Linux and macOS have a reader of the run memory',
25
+ },
26
+ });
27
+ }
28
+ return HOST_MEMORY.readRunMemory();
29
+ }
@@ -0,0 +1,14 @@
1
+ import type { HostMemory, RunMemoryReading } from './host-memory.js';
2
+ /** A file's text, or the errno code and message its read failed with. */
3
+ export type FileRead = {
4
+ readonly ok: true;
5
+ readonly text: string;
6
+ } | {
7
+ readonly ok: false;
8
+ readonly code: string;
9
+ readonly message: string;
10
+ };
11
+ /** The run's memory, reading every file through `read`. */
12
+ export declare function linuxRunMemory(read: (path: string) => Promise<FileRead>): Promise<RunMemoryReading>;
13
+ export declare const linuxMemory: HostMemory;
14
+ //# sourceMappingURL=host-memory.linux.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"host-memory.linux.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/host-memory.linux.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,UAAU,EAA6B,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAchG,yEAAyE;AACzE,MAAM,MAAM,QAAQ,GAChB;IAAE,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC;IAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GAC5C;IAAE,QAAQ,CAAC,EAAE,EAAE,KAAK,CAAC;IAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC;AA2B5E,2DAA2D;AAC3D,wBAAsB,cAAc,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,QAAQ,CAAC,GAAG,OAAO,CAAC,gBAAgB,CAAC,CA8CzG;AAoDD,eAAO,MAAM,WAAW,EAAE,UAA4F,CAAC"}