@smoothbricks/nx-plugin 0.4.29 → 0.4.31

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. package/README.md +104 -25
  2. package/dist/bin/smoo-nx-test-clocks.d.ts +3 -0
  3. package/dist/bin/smoo-nx-test-clocks.d.ts.map +1 -0
  4. package/dist/bin/smoo-nx-test-clocks.js +28 -0
  5. package/dist/bounded-test-policy.d.ts +1 -8
  6. package/dist/bounded-test-policy.d.ts.map +1 -1
  7. package/dist/bounded-test-policy.js +2 -8
  8. package/dist/cargo-closure-input.d.ts +11 -0
  9. package/dist/cargo-closure-input.d.ts.map +1 -1
  10. package/dist/cargo-closure-input.js +16 -6
  11. package/dist/ensure-built-receipt.d.ts +8 -2
  12. package/dist/ensure-built-receipt.d.ts.map +1 -1
  13. package/dist/ensure-built-receipt.js +45 -13
  14. package/dist/ensure-built-witness.d.ts +40 -4
  15. package/dist/ensure-built-witness.d.ts.map +1 -1
  16. package/dist/ensure-built-witness.js +68 -15
  17. package/dist/ensure-built.d.ts.map +1 -1
  18. package/dist/ensure-built.js +11 -3
  19. package/dist/executors/bounded-exec/cpu-pool.d.ts +7 -3
  20. package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -1
  21. package/dist/executors/bounded-exec/cpu-pool.js +9 -4
  22. package/dist/executors/bounded-exec/cpu-tokens.d.ts +22 -5
  23. package/dist/executors/bounded-exec/cpu-tokens.d.ts.map +1 -1
  24. package/dist/executors/bounded-exec/cpu-tokens.js +189 -18
  25. package/dist/executors/bounded-exec/executor.d.ts +1 -1
  26. package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
  27. package/dist/executors/bounded-exec/executor.js +38 -14
  28. package/dist/executors/bounded-exec/no-space.d.ts +8 -0
  29. package/dist/executors/bounded-exec/no-space.d.ts.map +1 -0
  30. package/dist/executors/bounded-exec/no-space.js +31 -0
  31. package/dist/executors/bounded-exec/ram-temp.d.ts +3 -0
  32. package/dist/executors/bounded-exec/ram-temp.d.ts.map +1 -1
  33. package/dist/executors/bounded-exec/ram-temp.js +15 -4
  34. package/dist/executors/bounded-exec/run-cpu-pool.d.ts +10 -0
  35. package/dist/executors/bounded-exec/run-cpu-pool.d.ts.map +1 -0
  36. package/dist/executors/bounded-exec/run-cpu-pool.js +63 -0
  37. package/dist/executors/bounded-exec/verdict.d.ts +34 -4
  38. package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
  39. package/dist/executors/bounded-exec/verdict.js +54 -3
  40. package/dist/graph-input-contract.d.ts +4 -0
  41. package/dist/graph-input-contract.d.ts.map +1 -1
  42. package/dist/graph-input-contract.js +17 -4
  43. package/dist/index.d.ts +3 -1
  44. package/dist/index.d.ts.map +1 -1
  45. package/dist/index.js +13 -3
  46. package/dist/owned-installation.d.ts +14 -0
  47. package/dist/owned-installation.d.ts.map +1 -0
  48. package/dist/owned-installation.js +324 -0
  49. package/dist/producer-receipts.d.ts.map +1 -1
  50. package/dist/producer-receipts.js +15 -10
  51. package/dist/test-backstop.d.ts +9 -0
  52. package/dist/test-backstop.d.ts.map +1 -0
  53. package/dist/test-backstop.js +10 -0
  54. package/dist/test-clock-policy.d.ts +30 -0
  55. package/dist/test-clock-policy.d.ts.map +1 -0
  56. package/dist/test-clock-policy.js +686 -0
  57. package/dist/testing.d.ts.map +1 -1
  58. package/dist/testing.js +8 -1
  59. package/managed/raw/tooling/direnv/developer-links.ts +1 -8
  60. package/managed/raw/tooling/direnv/setup-environment.ts +13 -1
  61. package/package.json +15 -4
  62. package/src/__tests__/cargo-closure-fixture.ts +113 -0
  63. package/src/__tests__/counted-cargo.ts +8 -25
  64. package/src/__tests__/directory-change.ts +27 -0
  65. package/src/__tests__/owned-installation.ts +4 -164
  66. package/src/__tests__/past-tick.ts +12 -0
  67. package/src/bin/smoo-nx-test-clocks.ts +29 -0
  68. package/src/bounded-test-policy.ts +2 -8
  69. package/src/cargo-closure-input.property.test.ts +50 -0
  70. package/src/cargo-closure-input.test.ts +15 -112
  71. package/src/cargo-closure-input.ts +23 -6
  72. package/src/cargo-closure-key.test.ts +240 -0
  73. package/src/cargo-resolve-cache.test.ts +29 -7
  74. package/src/ensure-built-receipt.ts +56 -13
  75. package/src/ensure-built-witness.test.ts +113 -4
  76. package/src/ensure-built-witness.ts +75 -16
  77. package/src/ensure-built.test.ts +74 -2
  78. package/src/ensure-built.ts +13 -2
  79. package/src/executors/bounded-exec/cpu-pool.test.ts +18 -0
  80. package/src/executors/bounded-exec/cpu-pool.ts +10 -4
  81. package/src/executors/bounded-exec/cpu-tokens.test.ts +205 -0
  82. package/src/executors/bounded-exec/cpu-tokens.ts +229 -22
  83. package/src/executors/bounded-exec/disk-lease.test.ts +47 -13
  84. package/src/executors/bounded-exec/executor-host.test.ts +23 -11
  85. package/src/executors/bounded-exec/executor.test.ts +189 -63
  86. package/src/executors/bounded-exec/executor.ts +39 -13
  87. package/src/executors/bounded-exec/hang-diagnostics.test.ts +14 -8
  88. package/src/executors/bounded-exec/no-space.ts +40 -0
  89. package/src/executors/bounded-exec/ram-temp-host.test.ts +9 -8
  90. package/src/executors/bounded-exec/ram-temp.test.ts +32 -18
  91. package/src/executors/bounded-exec/ram-temp.ts +17 -4
  92. package/src/executors/bounded-exec/run-cpu-pool.test.ts +76 -0
  93. package/src/executors/bounded-exec/run-cpu-pool.ts +75 -0
  94. package/src/executors/bounded-exec/verdict.test.ts +125 -25
  95. package/src/executors/bounded-exec/verdict.ts +78 -3
  96. package/src/graph-input-contract.test.ts +8 -0
  97. package/src/graph-input-contract.ts +26 -4
  98. package/src/index.test.ts +12 -2
  99. package/src/index.ts +13 -3
  100. package/src/nextest-extraction.test.ts +114 -4
  101. package/src/nx-daemon-claim.test.ts +5 -3
  102. package/src/nx-daemon-idle-policy.test.ts +22 -18
  103. package/src/nx-daemon-validity.test.ts +5 -13
  104. package/src/owned-installation.ts +174 -0
  105. package/src/producer-receipts.ts +15 -10
  106. package/src/test-backstop.ts +11 -0
  107. package/src/test-clock-policy.test.ts +338 -0
  108. package/src/test-clock-policy.ts +708 -0
  109. package/src/testing.ts +8 -1
  110. package/test-timings.json +4 -0
@@ -23,6 +23,7 @@ import {
23
23
  } from './cpu-tokens.js';
24
24
  import type { HangDiagnostics } from './hang-diagnostics.js';
25
25
  import { captureHangDiagnostics, HANG_CAPTURE_MS } from './hang-diagnostics.js';
26
+ import { createNoSpaceWatch } from './no-space.js';
26
27
  import { createRunProgress, withProgressReport } from './progress.js';
27
28
  import {
28
29
  COWSHED_EXECUTABLE_ENV,
@@ -38,11 +39,13 @@ import {
38
39
  type BoundedExecRecord,
39
40
  type BoundedExecTask,
40
41
  clearTaskDirectory,
42
+ describeNoSpace,
41
43
  type ExpiredBound,
42
44
  JUNIT_FILE,
43
45
  JUNIT_REPORT_ENV,
44
46
  judgeOperationalFailure,
45
47
  judgeRun,
48
+ type NoSpaceSighting,
46
49
  readJunitReport,
47
50
  taskDirectory,
48
51
  withJunitReport,
@@ -92,6 +95,8 @@ interface RunState {
92
95
  forceKillNeeded: boolean;
93
96
  /** The sub-pool's close error when it did not close cleanly: the run fails. */
94
97
  poolCloseError: string | null;
98
+ /** The first output line that said ENOSPC, with the task's volume read when it arrived. */
99
+ noSpace: Promise<NoSpaceSighting> | null;
95
100
  }
96
101
 
97
102
  type GrantedTokens = Extract<TokenGrant, { granted: true }>;
@@ -101,7 +106,7 @@ export interface ProcessTreeKiller {
101
106
  }
102
107
 
103
108
  /** Where the command's TMPDIR comes from; `null` leaves the inherited one. */
104
- export type TempVolume = Pick<RamTempVolume, 'acquire' | 'release' | 'fullness' | 'reap'>;
109
+ export type TempVolume = Pick<RamTempVolume, 'acquire' | 'release' | 'fullness' | 'space' | 'reap'>;
105
110
 
106
111
  /** Workspace-provided cowshed launchers, in the order a workspace that provides one names them. */
107
112
  const WORKSPACE_COWSHED_LAUNCHERS = [
@@ -248,6 +253,7 @@ export async function runBoundedExec(
248
253
  expiry: null,
249
254
  forceKillNeeded: false,
250
255
  poolCloseError: null,
256
+ noSpace: null,
251
257
  };
252
258
  const appendStdout = (chunk: Buffer | string): void => {
253
259
  const text = chunk.toString();
@@ -308,14 +314,15 @@ export async function runBoundedExec(
308
314
  }
309
315
  };
310
316
 
311
- // The wait is one span line, as cowshed's own spans read. With no gateway listening, as on any
312
- // host without cowshed, the command runs exactly as asked and nothing is said.
317
+ // The wait is one span line, as cowshed's own spans read. On a host without a gateway the Nx run serves its own
318
+ // pool (`run-cpu-pool.ts`), taken below as a parent's; a command no pool and no gateway answers runs exactly as
319
+ // asked and nothing is said.
313
320
  const cores = availableParallelism();
314
321
  const demand = runnerDemand(requested, options.parallelism, cores, process.env, options.env);
315
322
  const asked = Date.now();
316
- // A run a parent's CPU pool lent tokens to takes them there. Only an absent pool leaves the host budget to ask.
317
- // A pool that answered and refused still holds tokens of its own: asking the host then would hold them while waiting
318
- // on another budget, so the run fails instead.
323
+ // A run a parent's CPU pool serves takes its tokens there. Only an absent pool leaves the host budget to ask.
324
+ // A pool that answered and refused is a live budget of its own -- a lent grant, or a gate's or a resident's
325
+ // relay to the host -- so asking the host beside it would count the run twice; the run fails instead.
319
326
  let grant: GrantedTokens | null = null;
320
327
  let parentRefusal: string | null = null;
321
328
  if (inherited !== null) {
@@ -377,6 +384,7 @@ export async function runBoundedExec(
377
384
  expiry: ExpiredBound | null,
378
385
  elapsedMs: number,
379
386
  operational: boolean,
387
+ noSpace: NoSpaceSighting | null,
380
388
  ): Promise<void> => {
381
389
  if (context.record === undefined) {
382
390
  return;
@@ -385,7 +393,7 @@ export async function runBoundedExec(
385
393
  const report = await readJunitReport(recordDirectory);
386
394
  const verdict = operational
387
395
  ? judgeOperationalFailure(exitCode, report)
388
- : judgeRun({ exitCode, elapsedMs, expiry, report });
396
+ : judgeRun({ exitCode, elapsedMs, expiry, report, noSpace });
389
397
  if (verdict.outcome === 'bound' && verdict.bound === 'test') {
390
398
  appendStderr(`Every failing test failed only on its per-test timeout: ${verdict.tests.join(', ')}\n`);
391
399
  }
@@ -396,7 +404,7 @@ export async function runBoundedExec(
396
404
  appendStderr(message);
397
405
  await releaseCpu();
398
406
  await releaseLease();
399
- await recordRun(1, null, Date.now() - asked, false);
407
+ await recordRun(1, null, Date.now() - asked, false, null);
400
408
  return { success: false, terminalOutput: outputChunks.join('') };
401
409
  };
402
410
  if (parentRefusal !== null) {
@@ -461,6 +469,13 @@ export async function runBoundedExec(
461
469
 
462
470
  // Progress is a finished test, not output: a runner's SLOW warnings and a hung test's heartbeat keep printing.
463
471
  const progress = createRunProgress();
472
+ // A write that found no space is said once, in the output, by whatever made it; the volume is read when it is said,
473
+ // since the writer usually removes what it half-wrote before it exits and the volume no longer looks full by then.
474
+ const noSpace = createNoSpaceWatch();
475
+ const sight = async (line: string): Promise<NoSpaceSighting> => {
476
+ const held = lease;
477
+ return { line, volume: held === null ? null : ((await tempVolume?.space(held)) ?? null) };
478
+ };
464
479
  let idleTimer: NodeJS.Timeout | undefined;
465
480
  let captureTimer: NodeJS.Timeout | undefined;
466
481
  let idleCapture: { expiresAtMs: number; controller: AbortController; result: IdleDiagnosticResult } | undefined;
@@ -569,11 +584,15 @@ export async function runBoundedExec(
569
584
  child.stdout?.on('data', (chunk: Buffer | string) => {
570
585
  const text = chunk.toString();
571
586
  if (progress.scan('stdout', text)) armIdleTimer();
587
+ const said = noSpace.scan('stdout', text);
588
+ if (said !== null) state.noSpace = sight(said);
572
589
  appendStdout(text);
573
590
  });
574
591
  child.stderr?.on('data', (chunk: Buffer | string) => {
575
592
  const text = chunk.toString();
576
593
  if (progress.scan('stderr', text)) armIdleTimer();
594
+ const said = noSpace.scan('stderr', text);
595
+ if (said !== null) state.noSpace = sight(said);
577
596
  appendStderr(text);
578
597
  });
579
598
 
@@ -752,17 +771,24 @@ export async function runBoundedExec(
752
771
  appendStderr(`Timed out command exited after graceful termination: ${command}\n`);
753
772
  }
754
773
 
755
- // A test that dies of ENOSPC rarely says where it was writing; the volume can.
756
- const full = code !== 0 && lease !== null ? await tempVolume?.fullness(lease) : null;
757
- if (full) {
758
- appendStderr(`${describeRamTempError(full)}\n`);
774
+ // ENOSPC said in the output names the writer's volume as it stood then. Without it, a volume still (nearly) full now
775
+ // is the likely cause, and the volume is named.
776
+ const seen = state.noSpace === null ? null : await state.noSpace;
777
+ if (seen !== null && (code !== 0 || state.expiry !== null)) {
778
+ const writer = context.record?.task.id;
779
+ appendStderr(`${writer === undefined ? '' : `${writer}: `}${describeNoSpace(seen)}\n`);
780
+ } else {
781
+ const full = code !== 0 && lease !== null ? await tempVolume?.fullness(lease) : null;
782
+ if (full) {
783
+ appendStderr(`${describeRamTempError(full)}\n`);
784
+ }
759
785
  }
760
786
  await releaseLease();
761
787
 
762
788
  // A run whose sub-pool did not close cleanly fails even when the command exited 0, and whatever bound also fired:
763
789
  // its verdict is an ordinary failure, never a bound.
764
790
  const operationalFailure = state.poolCloseError !== null;
765
- await recordRun(code, state.expiry, Date.now() - startedAt, operationalFailure);
791
+ await recordRun(code, state.expiry, Date.now() - startedAt, operationalFailure, seen);
766
792
 
767
793
  return {
768
794
  success: !operationalFailure && state.expiry === null && code === 0,
@@ -1,7 +1,8 @@
1
- import { describe, expect, it } from 'bun:test';
1
+ import { describe, expect, it, jest } from 'bun:test';
2
2
  import { mkdtemp, rm, writeFile } from 'node:fs/promises';
3
3
  import { tmpdir } from 'node:os';
4
4
  import { join } from 'node:path';
5
+ import { BOUNDED_TEST_IDLE_TIMEOUT_MS } from '../../bounded-test-policy.js';
5
6
  import {
6
7
  captureHangDiagnostics,
7
8
  type HangCaptureRequest,
@@ -23,8 +24,8 @@ const exited = (stdout: string, code = 0, stderr = ''): HangProbeResult => ({ st
23
24
  const request = (): HangCaptureRequest => ({
24
25
  pid: ROOT,
25
26
  pgid: ROOT,
26
- expiresAtMs: Date.now() + 5_000,
27
- deadlineMs: 5_000,
27
+ expiresAtMs: Date.now() + BOUNDED_TEST_IDLE_TIMEOUT_MS,
28
+ deadlineMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
28
29
  });
29
30
 
30
31
  function scriptedHost(results: readonly HangProbeResult[]) {
@@ -140,6 +141,7 @@ describe('pre-expiry hang diagnostics', () => {
140
141
  return new Promise<HangProbeResult>(() => {});
141
142
  },
142
143
  };
144
+ // The probe is held off by construction, so the 1 ms deadline timer is the only event that can end this capture.
143
145
  const capture = await captureHangDiagnostics({ ...request(), deadlineMs: 1 }, unresponsive);
144
146
  expect(capture.outcome).toBe('incomplete');
145
147
  expect(capture.text).toContain('1ms deadline reached during ps before');
@@ -228,12 +230,11 @@ describe('pre-expiry hang diagnostics', () => {
228
230
  let descendant: number | undefined;
229
231
  const failures: unknown[] = [];
230
232
  try {
231
- const reader = child.stdout
232
- .pipeThrough(new TextDecoderStream(), { signal: AbortSignal.timeout(10_000) })
233
- .getReader();
233
+ const reader = child.stdout.pipeThrough(new TextDecoderStream()).getReader();
234
234
  let text = '';
235
235
  try {
236
- // Real OS-pipe readiness cannot advance a fake test clock; abort only bounds a missing line.
236
+ // The first complete line is the readiness event; a child that exits first ends the stream instead. No timer
237
+ // ends this loop.
237
238
  for (;;) {
238
239
  const chunk = await reader.read();
239
240
  if (chunk.done) throw new Error('child stdout ended before a complete PID line');
@@ -252,7 +253,12 @@ describe('pre-expiry hang diagnostics', () => {
252
253
  Number.isSafeInteger(descendant) && descendant > 0,
253
254
  `expected a tagged PID line, observed ${JSON.stringify(line)}`,
254
255
  ).toBe(true);
255
- const capture = await captureHangDiagnostics({ ...request(), pid: child.pid, pgid: null });
256
+ // The capture budget (`HANG_CAPTURE_MS`) and probe limits are product constants this test cannot set, so the
257
+ // clock is stopped while the real `ps` and `lsof` run: each ends when it exits; no timer cuts it short.
258
+ jest.useFakeTimers();
259
+ const capture = await captureHangDiagnostics({ ...request(), pid: child.pid, pgid: null }).finally(() =>
260
+ jest.useRealTimers(),
261
+ );
256
262
  expect(capture.outcome).toBe('holders');
257
263
  expect(capture.text).toContain(String(child.pid));
258
264
  expect(capture.text).toContain(String(descendant));
@@ -0,0 +1,40 @@
1
+ import type { OutputStream } from './progress.js';
2
+
3
+ /**
4
+ * The operating system's own words for ENOSPC: the errno name Node and Bun print (`ENOSPC: no space left on device,
5
+ * write`), and `strerror(28)`, which Rust (`No space left on device (os error 28)`), Go and C print. Nothing else in a
6
+ * command's output says the disk was full, and a write that failed on it is otherwise reported as whatever its caller
7
+ * was doing: a failed test, an extraction error.
8
+ */
9
+ const NO_SPACE = /\bENOSPC\b|No space left on device/;
10
+ const ANSI = new RegExp(`${String.fromCharCode(27)}\\[[0-9;]*[A-Za-z]`, 'g');
11
+
12
+ /** Watches a command's output for the first line that says a write found no space. */
13
+ export interface NoSpaceWatch {
14
+ /** One chunk of one output stream: the first matching line, once, as printed; `null` before and after it. */
15
+ scan(stream: OutputStream, chunk: string): string | null;
16
+ }
17
+
18
+ export function createNoSpaceWatch(): NoSpaceWatch {
19
+ const partial: Record<OutputStream, string> = { stdout: '', stderr: '' };
20
+ let seen = false;
21
+ return {
22
+ scan(stream, chunk) {
23
+ if (seen) {
24
+ return null;
25
+ }
26
+ const lines = (partial[stream] + chunk).split('\n');
27
+ // The unfinished last line is kept for the next chunk, and read now too: a command that dies mid-line still
28
+ // printed the words.
29
+ partial[stream] = lines.at(-1) ?? '';
30
+ for (const line of lines) {
31
+ const text = line.replace(ANSI, '').trim();
32
+ if (NO_SPACE.test(text)) {
33
+ seen = true;
34
+ return text;
35
+ }
36
+ }
37
+ return null;
38
+ },
39
+ };
40
+ }
@@ -16,10 +16,11 @@ import {
16
16
  } from 'node:fs';
17
17
  import { tmpdir, userInfo } from 'node:os';
18
18
  import { join } from 'node:path';
19
- import { setTimeout as sleep } from 'node:timers/promises';
20
19
  import { fileURLToPath } from 'node:url';
21
20
  import type { OpContextBinding } from '@smoothbricks/lmao';
22
21
  import { useTestSpan } from '@smoothbricks/lmao/testing/bun';
22
+ import { whenChanged } from '../../__tests__/directory-change.js';
23
+ import { BOUNDED_TEST_IDLE_TIMEOUT_MS } from '../../bounded-test-policy.js';
23
24
  import { processTable, terminate } from '../../testing.js';
24
25
  import {
25
26
  type CommandOutput,
@@ -386,9 +387,8 @@ async function nativeRamVolume(): Promise<void> {
386
387
  mkdirSync(quiet);
387
388
  writeFileSync(join(quiet, 'left'), 'by a task nothing of which still runs');
388
389
  try {
389
- while (!existsSync(join(leaseB.directory, 'w1'))) {
390
- await sleep(5);
391
- }
390
+ // Ends on the watch event that creates the writer's `w1` in the lease, not on a poll.
391
+ await whenChanged(leaseB.directory, () => existsSync(join(leaseB.directory, 'w1')));
392
392
  const fourth = new RamTempVolume(paths, capacity, commands, 101, (pid) => alive.has(pid), processes);
393
393
  const d = await fourth.acquire();
394
394
  if (!d.ok || d.value.kind !== 'leased') {
@@ -476,11 +476,12 @@ async function aFailedStepLeavesNoNamedVolume(): Promise<void> {
476
476
  (call) => call.file === '/usr/bin/hdiutil' && call.args[0] === 'detach' && call.output.status !== 0,
477
477
  );
478
478
  if (failedDetach !== undefined) {
479
- const mountTable = await host.run('/sbin/mount', [], 1_000);
479
+ // Diagnostic probes explain a failed detach: each waits for the host to answer, bounded by the leg backstop.
480
+ const mountTable = await host.run('/sbin/mount', [], BOUNDED_TEST_IDLE_TIMEOUT_MS);
480
481
  const filesystemHolders = await host.run(
481
482
  '/usr/bin/sudo',
482
483
  ['-n', '/usr/sbin/lsof', '-nP', '+f', '--', paths.staging],
483
- 1_000,
484
+ BOUNDED_TEST_IDLE_TIMEOUT_MS,
484
485
  );
485
486
  const added = calls.find(
486
487
  (call) => call.file === '/usr/sbin/diskutil' && call.args[0] === 'apfs' && call.args[1] === 'addVolume',
@@ -494,7 +495,7 @@ async function aFailedStepLeavesNoNamedVolume(): Promise<void> {
494
495
  const deviceHolders = await host.run(
495
496
  '/usr/bin/sudo',
496
497
  ['-n', '/usr/sbin/lsof', '-nP', '--', ...devicePaths],
497
- 1_000,
498
+ BOUNDED_TEST_IDLE_TIMEOUT_MS,
498
499
  );
499
500
  const observations = JSON.stringify(
500
501
  {
@@ -577,7 +578,7 @@ async function inheritedTempDirectory(): Promise<void> {
577
578
  for (const directory of [inherited, supplied]) {
578
579
  const options = {
579
580
  command: `node ${JSON.stringify(probe)}`,
580
- timeoutMs: 5_000,
581
+ timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
581
582
  parallelism: 1,
582
583
  env: {
583
584
  // If the default runner incorrectly provisions an APFS volume here, it must fail,
@@ -7,7 +7,8 @@ import { createServer, type Socket } from 'node:net';
7
7
  import { tmpdir } from 'node:os';
8
8
  import { join } from 'node:path';
9
9
  import { createInterface } from 'node:readline';
10
- import { setTimeout as sleep } from 'node:timers/promises';
10
+ import { whenChanged } from '../../__tests__/directory-change.js';
11
+ import { BOUNDED_TEST_IDLE_TIMEOUT_MS } from '../../bounded-test-policy.js';
11
12
  import { awaitExit, ownedSocketDirectory, pidsWorkingIn, processTable, terminate } from '../../testing.js';
12
13
  import {
13
14
  COWSHED_EXECUTABLE_ENV,
@@ -180,15 +181,29 @@ async function ownedDiskFacadeFixture() {
180
181
 
181
182
  describe('RAM temp volume on this host', () => {
182
183
  it('kills a deadlined command together with everything it spawned', async () => {
183
- // Ordinary helpers remain deadline-bound; disk-image tools are owned by cowshed instead.
184
- const result = await createHostCommands(500).run('/bin/sh', ['-c', 'sleep 60 & echo $!; wait']);
185
- expect(result.status).not.toBe(0);
186
- expect(result.stderr).toMatch(/^no answer within 500 ms; killed process group \d+$/);
187
- const spawned = result.stdout.trim();
188
- expect(spawned).toMatch(/^\d+$/);
189
- // Gone, or a zombie its new parent has yet to reap: either way no longer running.
190
- const state = await createHostCommands().run('ps', ['-o', 'stat=', '-p', spawned]);
191
- expect(state.status === 1 || state.stdout.trim().startsWith('Z')).toBe(true);
184
+ // Ordinary helpers remain deadline-bound; disk-image tools are owned by cowshed instead. The command never answers:
185
+ // its shell opens `lifetime`, hands it to a background `tail -f /dev/null` that never ends on its own, closes its
186
+ // own copy and only then prints the tail's pid. So only the deadline ends the run, a printed pid means the FIFO
187
+ // has a writer, and EOF on it is that descendant's death. The bound returns once it has signalled the group, not
188
+ // once the kernel has torn it down, so a `ps` read straight after raced the kill (Linux CI saw it still running).
189
+ const directory = mkdtempSync(join(realpathSync(tmpdir()), 'deadline-group-'));
190
+ try {
191
+ const lifetime = join(directory, 'lifetime.fifo');
192
+ execFileSync('mkfifo', [lifetime]);
193
+ const ended = readFile(lifetime, 'utf8');
194
+ const result = await createHostCommands(500).run('/bin/sh', [
195
+ '-c',
196
+ 'exec 3>"$1"; tail -f /dev/null & exec 3>&-; echo $!; wait',
197
+ 'sh',
198
+ lifetime,
199
+ ]);
200
+ expect(result.status).not.toBe(0);
201
+ expect(result.stderr).toMatch(/^no answer within 500 ms; killed process group \d+$/);
202
+ expect(result.stdout.trim()).toMatch(/^\d+$/);
203
+ expect(await ended).toBe('');
204
+ } finally {
205
+ rmSync(directory, { recursive: true, force: true });
206
+ }
192
207
  });
193
208
 
194
209
  it('routes disk tools and their lifetime together without shell interpolation', () => {
@@ -302,9 +317,9 @@ describe('RAM temp volume on this host', () => {
302
317
  const directory = await ownedSocketDirectory('rt-', 'missing-cowshed');
303
318
  try {
304
319
  const executable = join(directory, 'missing-cowshed');
305
- const output = await createHostCommands(300, undefined, undefined, executable).run('/usr/sbin/diskutil', [
306
- 'info',
307
- ]);
320
+ // The bound is the leg backstop: a missing facade fails at spawn, so the run ends at once.
321
+ const host = createHostCommands(BOUNDED_TEST_IDLE_TIMEOUT_MS, undefined, undefined, executable);
322
+ const output = await host.run('/usr/sbin/diskutil', ['info']);
308
323
  expect(output.status).toBe(1);
309
324
  expect(output.stdout).toBe('');
310
325
  expect(output.stderr).toContain(executable);
@@ -335,7 +350,7 @@ describe('RAM temp volume on this host', () => {
335
350
  server.listen(socket, listened);
336
351
  await listening;
337
352
  server.off('error', failed);
338
- const host = createHostCommands(10_000, socket);
353
+ const host = createHostCommands(BOUNDED_TEST_IDLE_TIMEOUT_MS, socket);
339
354
  // `umount` with no operand only prints its usage, and it is a disk tool that takes a lease.
340
355
  expect((await host.run('/sbin/umount', [])).status).not.toBe(0);
341
356
  expect((await host.run('/sbin/umount', [])).status).not.toBe(0);
@@ -369,7 +384,7 @@ describe('RAM temp volume on this host', () => {
369
384
  server.listen(socket, listened);
370
385
  await listening;
371
386
  server.off('error', failed);
372
- const host = createHostCommands(10_000, socket, 'alone');
387
+ const host = createHostCommands(BOUNDED_TEST_IDLE_TIMEOUT_MS, socket, 'alone');
373
388
  // `umount` with no operand only prints its usage, and it is a disk tool that takes a lease.
374
389
  expect((await host.run('/sbin/umount', [])).status).not.toBe(0);
375
390
  expect((await host.run('/sbin/umount', [])).status).not.toBe(0);
@@ -684,9 +699,8 @@ describe('processes a task left working in its RAM lease', () => {
684
699
  let child = 0;
685
700
  try {
686
701
  child = Number(String((await once(writer.stdout, 'data'))[0]).trim());
687
- while (readdirSync(lease).length < 2) {
688
- await sleep(5);
689
- }
702
+ // Ends on the watch event that brings the lease to two entries: `nested`, and the writer's first file.
703
+ await whenChanged(lease, () => readdirSync(lease).length >= 2);
690
704
  // None of them works in the lease by its working directory.
691
705
  expect(await pidsWorkingIn(lease)).toEqual([]);
692
706
 
@@ -39,6 +39,7 @@ import {
39
39
  parseHeldVerdict,
40
40
  validateHeldVerdict,
41
41
  } from './ram-temp-held.js';
42
+ import type { TempVolumeSpace } from './verdict.js';
42
43
 
43
44
  /**
44
45
  * One RAM-backed APFS volume per host user, shared by every test task that runs
@@ -856,12 +857,24 @@ export class RamTempVolume {
856
857
 
857
858
  /** After a failed task: whether the volume ran out of space, which is then the error to report. */
858
859
  async fullness(lease: RamTempLease): Promise<RamTempError | null> {
860
+ const space = await this.space(lease);
861
+ if (space === null || space.freeBytes >= FULL_FREE_BYTES) {
862
+ return null;
863
+ }
864
+ const { mountpoint, capacityBytes, freeBytes } = space;
865
+ return { kind: 'volume-full', mountpoint, capacityBytes, freeBytes };
866
+ }
867
+
868
+ /** The lease's volume and the space left on it now; `null` when the volume cannot be read. */
869
+ async space(lease: RamTempLease): Promise<TempVolumeSpace | null> {
859
870
  try {
860
871
  const fs = await statfs(lease.mountpoint);
861
- const freeBytes = fs.bavail * fs.bsize;
862
- return freeBytes < FULL_FREE_BYTES
863
- ? { kind: 'volume-full', mountpoint: lease.mountpoint, capacityBytes: lease.capacityBytes, freeBytes }
864
- : null;
872
+ return {
873
+ lease: lease.directory,
874
+ mountpoint: lease.mountpoint,
875
+ capacityBytes: lease.capacityBytes,
876
+ freeBytes: fs.bavail * fs.bsize,
877
+ };
865
878
  } catch {
866
879
  return null;
867
880
  }
@@ -0,0 +1,76 @@
1
+ import { afterEach, describe, expect, it } from 'bun:test';
2
+ import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
3
+ import { tmpdir } from 'node:os';
4
+ import { join } from 'node:path';
5
+ import { CPU_POOL_ENV, readInheritedCpuPool, takeParentCpuTokens } from './cpu-pool.js';
6
+ import { CPU_TOKENS_ENV } from './cpu-tokens.js';
7
+ import { holdLease } from './gateway-lease.js';
8
+ import { closeRunCpuPool, serveRunCpuPool } from './run-cpu-pool.js';
9
+
10
+ const directories: string[] = [];
11
+
12
+ afterEach(() => {
13
+ for (const directory of directories.splice(0)) {
14
+ rmSync(directory, { recursive: true, force: true });
15
+ }
16
+ });
17
+
18
+ /** A gateway socket path in a fresh directory: absent unless the test makes it. */
19
+ function gatewayPath(): string {
20
+ const directory = mkdtempSync(join(tmpdir(), 'run-cpu-pool-'));
21
+ directories.push(directory);
22
+ return join(directory, 'gateway.sock');
23
+ }
24
+
25
+ describe('a run without a gateway', () => {
26
+ it('serves its tasks one pool of one token per core, so a second runner waits for the first', async () => {
27
+ const env: NodeJS.ProcessEnv = {};
28
+ await serveRunCpuPool({ id: 'run-477' }, env, gatewayPath(), 4);
29
+ const pool = readInheritedCpuPool(env);
30
+ if (pool === null) {
31
+ throw new Error(`the run published no pool: ${JSON.stringify(env)}`);
32
+ }
33
+ expect(pool.tokens).toBe(4);
34
+ // Nightly 477's shape: two mutation sweeps, each asking for the whole host. The first holds the run's cores;
35
+ // the second queues for them instead of starting a second host's worth of workers beside it.
36
+ const first = await takeParentCpuTokens(pool, 31);
37
+ expect(first.granted && first.tokens).toBe(4);
38
+ const { promise: queued, resolve: queue } = Promise.withResolvers<void>();
39
+ const second = holdLease(
40
+ pool.socket,
41
+ { op: 'cpu-pool', want: 31 },
42
+ 'the run CPU pool',
43
+ { ackMs: 2_000, afterCloseMs: 5_000, grantMs: null },
44
+ queue,
45
+ );
46
+ let secondSettled = false;
47
+ void second.then(() => {
48
+ secondSettled = true;
49
+ });
50
+ await queued;
51
+ expect(secondSettled).toBe(false);
52
+ if (first.granted) first.release();
53
+ const granted = await second;
54
+ expect(granted.granted && granted.answer.tokens).toBe(4);
55
+ if (granted.granted) granted.release();
56
+
57
+ await closeRunCpuPool({ id: 'run-477' });
58
+ // The run's end closes its pool: nothing is left to grant from it.
59
+ const late = await takeParentCpuTokens(pool, 1);
60
+ expect(late.granted === false && late.cause).toBe('absent');
61
+ });
62
+
63
+ it('leaves a run that inherits a pool to that pool', async () => {
64
+ const env: NodeJS.ProcessEnv = { [CPU_POOL_ENV]: '/parent/pool.sock', [CPU_TOKENS_ENV]: '2' };
65
+ await serveRunCpuPool({ id: 'nested' }, env, gatewayPath(), 4);
66
+ expect(env).toEqual({ [CPU_POOL_ENV]: '/parent/pool.sock', [CPU_TOKENS_ENV]: '2' });
67
+ });
68
+
69
+ it('leaves a run on a host whose gateway socket exists to the gateway, answering or not', async () => {
70
+ const gateway = gatewayPath();
71
+ writeFileSync(gateway, '');
72
+ const env: NodeJS.ProcessEnv = {};
73
+ await serveRunCpuPool({ id: 'gateway-host' }, env, gateway, 4);
74
+ expect(env).toEqual({});
75
+ });
76
+ });
@@ -0,0 +1,75 @@
1
+ import { lstat } from 'node:fs/promises';
2
+ import { availableParallelism } from 'node:os';
3
+ import { logger } from 'nx/src/devkit-exports.js';
4
+ import type { PostTasksExecutionContext, PreTasksExecutionContext } from 'nx/src/project-graph/plugins/public-api.js';
5
+ import { type CpuPool, createCpuPool, readInheritedCpuPool } from './cpu-pool.js';
6
+ import { GATEWAY_SOCKET } from './gateway-lease.js';
7
+
8
+ /**
9
+ * The CPU budget of an Nx run on a host without a gateway (specs/cowshed/05_gateway.md, "Host CPU budget", "A run
10
+ * without a gateway").
11
+ *
12
+ * Every runner sizes itself to the whole host unless something budgets it, and without a gateway nothing did: under
13
+ * `NX_PARALLEL=100%` Nx starts every target side by side and each one's runner takes the whole machine. The run
14
+ * therefore serves its own pool of one token per core, as a gateway-granted executor serves its children, and every
15
+ * task inherits it: `bounded-exec` and the `cargo` launcher take from it as from any parent pool, FIFO, the head
16
+ * taking `min(want, free)` once one token is free, so the runners of one run together stay within the cores.
17
+ *
18
+ * The run serves it only when nothing else budgets it: it inherits no pool, and the gateway's control socket does
19
+ * not exist. Only `ENOENT` proves the gateway absent, as for the launcher; a socket that exists but does not answer
20
+ * may be a live, saturated gateway, and a run budgeting itself beside it would count the host twice.
21
+ */
22
+
23
+ /** The pools this process serves, by Nx run id, until that run's `postTasksExecution`. */
24
+ const pools = new Map<string, CpuPool>();
25
+
26
+ /**
27
+ * Serve `context`'s run a pool of `cores` tokens and publish it in `env`, which every task inherits, when the run
28
+ * inherits no pool and no gateway exists at `gateway`. A pool that cannot be made is said and leaves the run
29
+ * unbudgeted, as a run without a gateway was before.
30
+ */
31
+ export async function serveRunCpuPool(
32
+ context: Pick<PreTasksExecutionContext, 'id'>,
33
+ env: NodeJS.ProcessEnv = process.env,
34
+ gateway: string = GATEWAY_SOCKET,
35
+ cores: number = availableParallelism(),
36
+ ): Promise<void> {
37
+ if (readInheritedCpuPool(env) !== null || !(await absent(gateway))) {
38
+ return;
39
+ }
40
+ // The pool lives until the run's `postTasksExecution` closes it, or with this process.
41
+ const made = await createCpuPool(cores, new Promise<void>(() => {}), { begin() {}, end() {} });
42
+ if (!made.ok) {
43
+ logger.warn(
44
+ `bounded-exec: no gateway at ${gateway}, and this run's CPU pool failed (${made.error}); its tasks run unbudgeted`,
45
+ );
46
+ return;
47
+ }
48
+ pools.set(context.id, made.pool);
49
+ for (const [name, value] of Object.entries(made.pool.environment)) {
50
+ env[name] = value;
51
+ }
52
+ }
53
+
54
+ /** Close the pool `context`'s run was served, if it was served one. */
55
+ export async function closeRunCpuPool(context: Pick<PostTasksExecutionContext, 'id'>): Promise<void> {
56
+ const pool = pools.get(context.id);
57
+ if (pool === undefined) {
58
+ return;
59
+ }
60
+ pools.delete(context.id);
61
+ const closed = await pool.close();
62
+ if (!closed.ok) {
63
+ logger.warn(`bounded-exec: this run's CPU pool did not close cleanly: ${closed.error}`);
64
+ }
65
+ }
66
+
67
+ async function absent(path: string): Promise<boolean> {
68
+ try {
69
+ await lstat(path);
70
+ return false;
71
+ } catch (error) {
72
+ const code = error instanceof Error && 'code' in error ? error.code : undefined;
73
+ return code === 'ENOENT' || code === 'ENOTDIR';
74
+ }
75
+ }