@smoothbricks/nx-plugin 0.4.29 → 0.4.30

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. package/README.md +83 -15
  2. package/dist/bin/smoo-nx-test-clocks.d.ts +3 -0
  3. package/dist/bin/smoo-nx-test-clocks.d.ts.map +1 -0
  4. package/dist/bin/smoo-nx-test-clocks.js +28 -0
  5. package/dist/bounded-test-policy.d.ts +1 -8
  6. package/dist/bounded-test-policy.d.ts.map +1 -1
  7. package/dist/bounded-test-policy.js +2 -8
  8. package/dist/cargo-closure-input.d.ts +11 -0
  9. package/dist/cargo-closure-input.d.ts.map +1 -1
  10. package/dist/cargo-closure-input.js +16 -6
  11. package/dist/ensure-built-receipt.d.ts +8 -2
  12. package/dist/ensure-built-receipt.d.ts.map +1 -1
  13. package/dist/ensure-built-receipt.js +45 -13
  14. package/dist/ensure-built-witness.d.ts +40 -4
  15. package/dist/ensure-built-witness.d.ts.map +1 -1
  16. package/dist/ensure-built-witness.js +68 -15
  17. package/dist/ensure-built.d.ts.map +1 -1
  18. package/dist/ensure-built.js +11 -3
  19. package/dist/executors/bounded-exec/cpu-pool.d.ts +7 -3
  20. package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -1
  21. package/dist/executors/bounded-exec/cpu-pool.js +9 -4
  22. package/dist/executors/bounded-exec/executor.d.ts +1 -1
  23. package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
  24. package/dist/executors/bounded-exec/executor.js +35 -12
  25. package/dist/executors/bounded-exec/no-space.d.ts +8 -0
  26. package/dist/executors/bounded-exec/no-space.d.ts.map +1 -0
  27. package/dist/executors/bounded-exec/no-space.js +31 -0
  28. package/dist/executors/bounded-exec/ram-temp.d.ts +3 -0
  29. package/dist/executors/bounded-exec/ram-temp.d.ts.map +1 -1
  30. package/dist/executors/bounded-exec/ram-temp.js +15 -4
  31. package/dist/executors/bounded-exec/verdict.d.ts +34 -4
  32. package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
  33. package/dist/executors/bounded-exec/verdict.js +54 -3
  34. package/dist/graph-input-contract.d.ts +4 -0
  35. package/dist/graph-input-contract.d.ts.map +1 -1
  36. package/dist/graph-input-contract.js +17 -4
  37. package/dist/owned-installation.d.ts +14 -0
  38. package/dist/owned-installation.d.ts.map +1 -0
  39. package/dist/owned-installation.js +324 -0
  40. package/dist/producer-receipts.d.ts.map +1 -1
  41. package/dist/producer-receipts.js +15 -10
  42. package/dist/test-backstop.d.ts +9 -0
  43. package/dist/test-backstop.d.ts.map +1 -0
  44. package/dist/test-backstop.js +10 -0
  45. package/dist/test-clock-policy.d.ts +30 -0
  46. package/dist/test-clock-policy.d.ts.map +1 -0
  47. package/dist/test-clock-policy.js +686 -0
  48. package/dist/testing.d.ts.map +1 -1
  49. package/dist/testing.js +8 -1
  50. package/managed/raw/tooling/direnv/developer-links.ts +1 -8
  51. package/managed/raw/tooling/direnv/setup-environment.ts +0 -1
  52. package/package.json +15 -4
  53. package/src/__tests__/cargo-closure-fixture.ts +113 -0
  54. package/src/__tests__/counted-cargo.ts +8 -25
  55. package/src/__tests__/directory-change.ts +27 -0
  56. package/src/__tests__/owned-installation.ts +4 -164
  57. package/src/__tests__/past-tick.ts +12 -0
  58. package/src/bin/smoo-nx-test-clocks.ts +29 -0
  59. package/src/bounded-test-policy.ts +2 -8
  60. package/src/cargo-closure-input.property.test.ts +50 -0
  61. package/src/cargo-closure-input.test.ts +15 -112
  62. package/src/cargo-closure-input.ts +23 -6
  63. package/src/cargo-closure-key.test.ts +239 -0
  64. package/src/cargo-resolve-cache.test.ts +29 -7
  65. package/src/ensure-built-receipt.ts +56 -13
  66. package/src/ensure-built-witness.test.ts +113 -4
  67. package/src/ensure-built-witness.ts +75 -16
  68. package/src/ensure-built.test.ts +74 -2
  69. package/src/ensure-built.ts +13 -2
  70. package/src/executors/bounded-exec/cpu-pool.test.ts +18 -0
  71. package/src/executors/bounded-exec/cpu-pool.ts +10 -4
  72. package/src/executors/bounded-exec/disk-lease.test.ts +47 -13
  73. package/src/executors/bounded-exec/executor-host.test.ts +23 -11
  74. package/src/executors/bounded-exec/executor.test.ts +189 -63
  75. package/src/executors/bounded-exec/executor.ts +36 -11
  76. package/src/executors/bounded-exec/hang-diagnostics.test.ts +14 -8
  77. package/src/executors/bounded-exec/no-space.ts +40 -0
  78. package/src/executors/bounded-exec/ram-temp-host.test.ts +9 -8
  79. package/src/executors/bounded-exec/ram-temp.test.ts +32 -18
  80. package/src/executors/bounded-exec/ram-temp.ts +17 -4
  81. package/src/executors/bounded-exec/verdict.test.ts +125 -25
  82. package/src/executors/bounded-exec/verdict.ts +78 -3
  83. package/src/graph-input-contract.test.ts +8 -0
  84. package/src/graph-input-contract.ts +26 -4
  85. package/src/nextest-extraction.test.ts +114 -4
  86. package/src/nx-daemon-claim.test.ts +5 -3
  87. package/src/nx-daemon-idle-policy.test.ts +22 -18
  88. package/src/nx-daemon-validity.test.ts +5 -13
  89. package/src/owned-installation.ts +174 -0
  90. package/src/producer-receipts.ts +15 -10
  91. package/src/test-backstop.ts +11 -0
  92. package/src/test-clock-policy.test.ts +338 -0
  93. package/src/test-clock-policy.ts +708 -0
  94. package/src/testing.ts +8 -1
  95. package/test-timings.json +3 -0
@@ -1,15 +1,17 @@
1
- import { afterEach, describe, expect, it } from 'bun:test';
1
+ import { afterEach, describe, expect, it, jest } from 'bun:test';
2
2
  import { spawnSync } from 'node:child_process';
3
3
  import { access, mkdir, mkdtemp, open, readFile, rm, writeFile } from 'node:fs/promises';
4
4
  import { createServer } from 'node:net';
5
5
  import { tmpdir } from 'node:os';
6
6
  import { dirname, join } from 'node:path';
7
7
  import { onProgressClock } from '../../__tests__/progress-clock.js';
8
+ import { BOUNDED_TEST_IDLE_TIMEOUT_MS } from '../../bounded-test-policy.js';
8
9
  import { ownedSocketDirectory } from '../../testing.js';
9
10
  import { CPU_POOL_ENV, type CpuPool, createCpuPool, type InheritedCpuPool, readInheritedCpuPool } from './cpu-pool.js';
10
11
  import { CPU_TOKENS_ENV, type CpuBudget, parseInheritedTokens, type TokenGrant } from './cpu-tokens.js';
11
12
  import {
12
13
  type BoundedExecContext,
14
+ type BoundedExecResult,
13
15
  createProcessTreeKiller,
14
16
  type ProcessTreeKiller,
15
17
  resolveCowshedExecutable,
@@ -17,8 +19,9 @@ import {
17
19
  type TempVolume,
18
20
  } from './executor.js';
19
21
  import type { HangDiagnostics } from './hang-diagnostics.js';
22
+ import { createNoSpaceWatch } from './no-space.js';
20
23
  import type { RamTempAcquisition, RamTempError, RamTempLease, Result } from './ram-temp.js';
21
- import { VERDICT_FILE } from './verdict.js';
24
+ import { readRecord, type TempVolumeSpace, VERDICT_FILE } from './verdict.js';
22
25
 
23
26
  const workspaces: string[] = [];
24
27
  const ownedProcessGroups = new Set<number>();
@@ -65,12 +68,14 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
65
68
  'console.log(process.cwd()); console.log(process.env.BOUNDED_EXEC_VALUE);\n',
66
69
  );
67
70
 
71
+ // bounded-exec requires a ceiling; at the leg's backstop it cannot end a run whose `node` is slow to start
72
+ // under load.
68
73
  const result = await runBoundedExec(
69
74
  {
70
75
  command: 'node print.js',
71
76
  cwd: 'package',
72
77
  env: { BOUNDED_EXEC_VALUE: 'from-env' },
73
- timeoutMs: 5_000,
78
+ timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
74
79
  },
75
80
  workspace.context,
76
81
  createProcessTreeKiller(),
@@ -87,7 +92,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
87
92
  const workspace = await createWorkspace();
88
93
 
89
94
  const result = await runBoundedExec(
90
- { command: 'node -e "process.exit(7)"', timeoutMs: 5_000 },
95
+ { command: 'node -e "process.exit(7)"', timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS },
91
96
  workspace.context,
92
97
  createProcessTreeKiller(),
93
98
  null,
@@ -106,7 +111,10 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
106
111
  process.stderr.write = captureWrite(stderr);
107
112
 
108
113
  const result = await runBoundedExec(
109
- { command: "node -e \"console.log('out-value'); console.error('err-value')\"", timeoutMs: 5_000 },
114
+ {
115
+ command: "node -e \"console.log('out-value'); console.error('err-value')\"",
116
+ timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
117
+ },
110
118
  workspace.context,
111
119
  createProcessTreeKiller(),
112
120
  null,
@@ -123,8 +131,9 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
123
131
  it('fails on timeout and reports bounded execution details', async () => {
124
132
  const workspace = await createWorkspace();
125
133
 
134
+ // The command never exits, so only the ceiling under test can end it.
126
135
  const result = await runBoundedExec(
127
- { command: 'node -e "setTimeout(() => {}, 5000)"', timeoutMs: 50, killAfterMs: 0 },
136
+ { command: 'node -e "process.stdin.resume()"', timeoutMs: 50, killAfterMs: 0 },
128
137
  workspace.context,
129
138
  createProcessTreeKiller(),
130
139
  null,
@@ -140,8 +149,9 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
140
149
  it('kills a silent command on the progress bound, with no ceiling at all', async () => {
141
150
  const workspace = await createWorkspace();
142
151
 
152
+ // The command never exits and never speaks, so only the progress bound under test can end it.
143
153
  const result = await runBoundedExec(
144
- { command: 'node -e "setTimeout(() => {}, 30000)"', idleTimeoutMs: 100, killAfterMs: 0 },
154
+ { command: 'node -e "process.stdin.resume()"', idleTimeoutMs: 100, killAfterMs: 0 },
145
155
  workspace.context,
146
156
  createProcessTreeKiller(),
147
157
  null,
@@ -435,7 +445,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
435
445
  await workspace.write('runaway.js', ["setInterval(() => console.log('still here'), 10);", ''].join('\n'));
436
446
 
437
447
  const result = await runBoundedExec(
438
- { command: 'node runaway.js', timeoutMs: 300, idleTimeoutMs: 30_000, killAfterMs: 0 },
448
+ { command: 'node runaway.js', timeoutMs: 300, idleTimeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS, killAfterMs: 0 },
439
449
  workspace.context,
440
450
  createProcessTreeKiller(),
441
451
  null,
@@ -450,17 +460,47 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
450
460
 
451
461
  it('imposes no progress bound when idleTimeoutMs is omitted', async () => {
452
462
  const workspace = await createWorkspace();
453
-
454
- const result = await runBoundedExec(
455
- { command: 'node -e "setTimeout(() => console.log(\'late\'), 400)"', timeoutMs: 30_000 },
456
- workspace.context,
457
- createProcessTreeKiller(),
458
- null,
459
- null,
463
+ const go = join(workspace.root, 'go.fifo');
464
+ expect(spawnSync('mkfifo', [go]).status).toBe(0);
465
+ // Says `started`, then goes silent until the test writes `go.fifo`.
466
+ await workspace.write(
467
+ 'silent.js',
468
+ [
469
+ "const fs = require('node:fs');",
470
+ "console.log('started');",
471
+ "fs.readFileSync('go.fifo');",
472
+ "console.log('late');",
473
+ '',
474
+ ].join('\n'),
460
475
  );
476
+ const { promise: started, resolve: sawStarted } = Promise.withResolvers<void>();
477
+ process.stdout.write = captureWrite([], (text) => {
478
+ if (text.includes('started')) sawStarted();
479
+ });
480
+ // Every bound is armed as the command spawns, before it can say `started`. The clock then moves to just short of
481
+ // the ceiling while the command is silent: any progress bound would have fired there, and the ceiling has not.
482
+ // Only then is the command released, so the run ends by its own exit.
483
+ jest.useFakeTimers();
484
+ let result: BoundedExecResult;
485
+ try {
486
+ const run = runBoundedExec(
487
+ { command: 'node silent.js', timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS },
488
+ workspace.context,
489
+ createProcessTreeKiller(),
490
+ null,
491
+ null,
492
+ );
493
+ await started;
494
+ jest.advanceTimersByTime(BOUNDED_TEST_IDLE_TIMEOUT_MS - 1);
495
+ await writeFile(go, 'go\n');
496
+ result = await run;
497
+ } finally {
498
+ jest.useRealTimers();
499
+ }
461
500
 
462
501
  expect(result.success).toBe(true);
463
502
  expect(result.terminalOutput).toContain('late');
503
+ expect(result.terminalOutput).not.toContain('made no progress');
464
504
  });
465
505
 
466
506
  it('uses graceful timeout termination before force-killing on POSIX', async () => {
@@ -481,7 +521,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
481
521
  ' process.exit(0);',
482
522
  '});',
483
523
  "fs.writeFileSync('cooperative-ready.fifo', 'ready pid=' + process.pid + '\\n');",
484
- 'setTimeout(() => {}, 5000);',
524
+ 'process.stdin.resume();',
485
525
  '',
486
526
  ].join('\n'),
487
527
  );
@@ -491,8 +531,10 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
491
531
  expect(handlerPid, `handler readiness must name the command leader pid=${pid}`).toBe(pid);
492
532
  });
493
533
 
534
+ // The ceiling is the subject; the grace after the graceful signal is the leg's backstop, so the command ends by its
535
+ // handler alone however slowly a loaded host runs it.
494
536
  const result = await runBoundedExec(
495
- { command: 'exec node cooperative.js', timeoutMs: 50, killAfterMs: 500 },
537
+ { command: 'exec node cooperative.js', timeoutMs: 50, killAfterMs: BOUNDED_TEST_IDLE_TIMEOUT_MS },
496
538
  workspace.context,
497
539
  termination.killer,
498
540
  null,
@@ -552,15 +594,18 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
552
594
  leaked = pid;
553
595
  },
554
596
  };
555
- const started = Date.now();
556
597
  const result = await runBoundedExec(
557
- { command: 'node -e "setTimeout(() => {}, 30000)"', timeoutMs: 50, killAfterMs: 10 },
598
+ { command: 'node -e "process.stdin.resume()"', timeoutMs: 50, killAfterMs: 10 },
558
599
  workspace.context,
559
600
  killer,
560
601
  null,
561
602
  null,
562
603
  );
563
- expect(Date.now() - started).toBeLessThan(5_000);
604
+ // The command never exits and the killer delivers nothing, so the run returned by giving up on the tree after its
605
+ // force-kill, not by the command ending: the command is still alive here.
606
+ if (leaked === undefined) throw new Error('the killer was never asked to kill the command');
607
+ const pid = leaked;
608
+ expect(() => process.kill(pid, 0)).not.toThrow();
564
609
  expect(result.success).toBe(false);
565
610
  expect(result.terminalOutput).toContain('Force-killing timed out command');
566
611
  if (leaked !== undefined) {
@@ -585,7 +630,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
585
630
  command: 'node -e "console.log(process.argv.slice(1).join(\',\'))"',
586
631
  args: ['first'],
587
632
  __unparsed__: ['second'],
588
- timeoutMs: 5_000,
633
+ timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
589
634
  },
590
635
  workspace.context,
591
636
  createProcessTreeKiller(),
@@ -598,7 +643,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
598
643
  args: ['first'],
599
644
  __unparsed__: ['second'],
600
645
  forwardAllArgs: false,
601
- timeoutMs: 5_000,
646
+ timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
602
647
  },
603
648
  workspace.context,
604
649
  createProcessTreeKiller(),
@@ -619,7 +664,6 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
619
664
  }
620
665
 
621
666
  const workspace = await createWorkspace();
622
- const marker = join(workspace.root, 'marker.txt');
623
667
  const ready = join(workspace.root, 'descendant-ready.fifo');
624
668
  const lifetime = join(workspace.root, 'descendant-lifetime.fifo');
625
669
  expect(spawnSync('mkfifo', [ready, lifetime]).status).toBe(0);
@@ -629,7 +673,8 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
629
673
  "const fs = require('node:fs');",
630
674
  "fs.openSync('descendant-lifetime.fifo', 'w');",
631
675
  "fs.writeFileSync('descendant-ready.fifo', 'ready pid=' + process.pid + '\\n');",
632
- "setTimeout(() => fs.writeFileSync(process.argv[2], 'alive'), 700);",
676
+ // Its stdin is /dev/null, so it holds itself: a wait no event ever ends, until a signal does.
677
+ 'Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0);',
633
678
  '',
634
679
  ].join('\n'),
635
680
  );
@@ -637,12 +682,12 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
637
682
  'spawn-child.js',
638
683
  [
639
684
  "import { spawn } from 'node:child_process';",
640
- "spawn(process.execPath, ['group-child.js', process.argv[2]], { stdio: 'ignore' });",
641
- 'setTimeout(() => {}, 5000);',
685
+ "spawn(process.execPath, ['group-child.js'], { stdio: 'ignore' });",
686
+ 'process.stdin.resume();',
642
687
  '',
643
688
  ].join('\n'),
644
689
  );
645
- // Only the descendant owns the writer; EOF observes its actual terminal closure, not a time horizon.
690
+ // Only the descendant owns the writer and it never exits on its own: EOF observes its kill, not a time horizon.
646
691
  const closed = readToEnd(lifetime);
647
692
  let descendantPid: number | undefined;
648
693
  const termination = armedKiller('process-group descendant', async (pid) => {
@@ -651,7 +696,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
651
696
  });
652
697
 
653
698
  const result = await runBoundedExec(
654
- { command: `exec node spawn-child.js ${marker}`, timeoutMs: 50, killAfterMs: 50 },
699
+ { command: 'exec node spawn-child.js', timeoutMs: 50, killAfterMs: 50 },
655
700
  workspace.context,
656
701
  termination.killer,
657
702
  null,
@@ -661,7 +706,6 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
661
706
  await termination.armed();
662
707
  expect(await closed, `ready descendant pid=${descendantPid} must close its lifetime FIFO on exit`).toBe('');
663
708
  expect(result.success).toBe(false);
664
- expect(await exists(marker)).toBe(false);
665
709
  });
666
710
 
667
711
  it('force-kills what outlives a timed-out command in its process group', async () => {
@@ -674,27 +718,26 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
674
718
  const ready = join(workspace.root, 'straggler-ready.fifo');
675
719
  expect(spawnSync('mkfifo', [fifo, ready]).status).toBe(0);
676
720
  // The leader exits on SIGTERM. The subshell beside it ignores SIGTERM, a disposition its
677
- // `sleep` inherits, and announces itself only once it holds the FIFO: from then on the
721
+ // `tail` inherits, and announces itself only once it holds the FIFO: from then on the
678
722
  // graceful signal cannot end it. A process forked while that signal goes out is the same
679
723
  // case, without the trap.
680
724
  await workspace.write(
681
725
  'straggle.sh',
682
726
  [
683
- '( trap "" TERM; exec 3>"$1"; echo ready; echo armed >"$2"; sleep 10; echo survived >&3 ) | {',
684
- ' read -r line; echo "$line"; exec sleep 30',
727
+ '( trap "" TERM; exec 3>"$1"; echo ready; echo armed >"$2"; exec tail -f /dev/null ) | {',
728
+ ' read -r line; echo "$line"; exec tail -f /dev/null',
685
729
  '}',
686
730
  '',
687
731
  ].join('\n'),
688
732
  );
689
- // The straggler is the FIFO's only writer, so the read ends when it is gone: with nothing
690
- // written if it was killed, with "survived" if it lived to finish its sleep.
733
+ // The straggler is the FIFO's only writer and never ends on its own, so the read ends only when it is killed.
691
734
  const straggler = readToEnd(fifo);
692
735
  const termination = armedKiller('TERM-ignoring FIFO straggler', async (pid) => {
693
736
  expect(await readToEnd(ready), `straggler readiness pgid=${pid} must follow trap and FIFO open`).toBe('armed\n');
694
737
  });
695
738
 
696
739
  const result = await runBoundedExec(
697
- { command: `sh straggle.sh ${fifo} ${ready}`, timeoutMs: 5_000, killAfterMs: 30_000 },
740
+ { command: `sh straggle.sh ${fifo} ${ready}`, timeoutMs: 5_000, killAfterMs: BOUNDED_TEST_IDLE_TIMEOUT_MS },
698
741
  workspace.context,
699
742
  termination.killer,
700
743
  null,
@@ -719,7 +762,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
719
762
  });
720
763
 
721
764
  const result = await runBoundedExec(
722
- { command: 'node -e "console.log(process.env.TMPDIR)"', timeoutMs: 5_000 },
765
+ { command: 'node -e "console.log(process.env.TMPDIR)"', timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS },
723
766
  workspace.context,
724
767
  createProcessTreeKiller(),
725
768
  volume,
@@ -746,7 +789,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
746
789
  );
747
790
 
748
791
  const result = await runBoundedExec(
749
- { command: 'node -e "0"', timeoutMs: 5_000 },
792
+ { command: 'node -e "0"', timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS },
750
793
  workspace.context,
751
794
  createProcessTreeKiller(),
752
795
  volume,
@@ -768,7 +811,11 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
768
811
  const volume = scriptedVolume({ ok: true, value: { kind: 'sandboxed', detail: 'lock: EPERM' } });
769
812
 
770
813
  const result = await runBoundedExec(
771
- { command: `node -e 'console.log("tmp=" + process.env.TMPDIR)'`, timeoutMs: 5_000, env: { TMPDIR: '/shed/tmp' } },
814
+ {
815
+ command: `node -e 'console.log("tmp=" + process.env.TMPDIR)'`,
816
+ timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
817
+ env: { TMPDIR: '/shed/tmp' },
818
+ },
772
819
  workspace.context,
773
820
  createProcessTreeKiller(),
774
821
  volume,
@@ -790,7 +837,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
790
837
  });
791
838
 
792
839
  const result = await runBoundedExec(
793
- { command: `touch ${marker}`, timeoutMs: 5_000 },
840
+ { command: `touch ${marker}`, timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS },
794
841
  workspace.context,
795
842
  createProcessTreeKiller(),
796
843
  volume,
@@ -812,7 +859,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
812
859
  );
813
860
 
814
861
  const result = await runBoundedExec(
815
- { command: 'node -e "process.exit(1)"', timeoutMs: 5_000 },
862
+ { command: 'node -e "process.exit(1)"', timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS },
816
863
  workspace.context,
817
864
  createProcessTreeKiller(),
818
865
  volume,
@@ -824,6 +871,48 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
824
871
  expect(volume.released).toEqual([directory]);
825
872
  });
826
873
 
874
+ it('names ENOSPC, its writer and the volume as it stood when the output said so, not after the command exited', async () => {
875
+ // From a downstream land gate: a compiler test shard printed ENOSPC for a 1 KB write into its lease and failed two.
876
+ // By its exit the volume had space again (`fullness` answers null here), so a probe after exit named nothing and
877
+ // the run read as two failed tests.
878
+ const workspace = await createWorkspace();
879
+ const directory = join(workspace.root, 'lease');
880
+ await mkdir(directory);
881
+ const lease = leaseAt(directory);
882
+ const space = { lease: directory, mountpoint: workspace.root, capacityBytes: lease.capacityBytes, freeBytes: 4096 };
883
+ const volume = scriptedVolume(
884
+ { ok: true, value: { kind: 'leased', lease, held: [], reaped: [] } },
885
+ null,
886
+ [],
887
+ space,
888
+ );
889
+ const record = { task: { id: 'compiler:test-10', hash: 'h' }, directory: join(workspace.root, 'record') };
890
+ const line = 'ENOSPC: no space left on device, write';
891
+
892
+ const result = await runBoundedExec(
893
+ { command: `node -e "console.error('${line}'); process.exit(1)"`, timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS },
894
+ { ...workspace.context, record },
895
+ createProcessTreeKiller(),
896
+ volume,
897
+ null,
898
+ );
899
+
900
+ expect(result.success).toBe(false);
901
+ expect(result.terminalOutput).toContain(
902
+ `compiler:test-10: ENOSPC (no space left on device): the command printed "${line}"; its TMPDIR ` +
903
+ `${directory} is on the RAM temp volume ${workspace.root}, which had 0.0 MiB free of 1024.0 MiB when that ` +
904
+ 'line arrived\n',
905
+ );
906
+ expect(result.terminalOutput).not.toContain('is full:');
907
+ expect(await readRecord(record.directory)).toEqual({
908
+ task: record.task.id,
909
+ hash: 'h',
910
+ verdict: { outcome: 'no-space', exitCode: 1, tests: [], line, volume: space },
911
+ });
912
+ // Read while the lease was still the task's, before it was reaped and released.
913
+ expect(volume.order).toEqual(['space', `reap ${directory}`, `release ${directory}`]);
914
+ });
915
+
827
916
  it('keeps inherited nextest and Cargo ceilings under oversized grants', async () => {
828
917
  const previousNextest = process.env.NEXTEST_TEST_THREADS;
829
918
  const previousCargo = process.env.CARGO_BUILD_JOBS;
@@ -850,7 +939,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
850
939
  const result = await runBoundedExec(
851
940
  {
852
941
  command,
853
- timeoutMs: 5_000,
942
+ timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
854
943
  parallelism: 18,
855
944
  env: { NEXTEST_TEST_THREADS: '64', CARGO_BUILD_JOBS: '64', RUST_TEST_THREADS: '64' },
856
945
  },
@@ -890,7 +979,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
890
979
  "node -e \"console.log('threads=' + process.env.NEXTEST_TEST_THREADS + ' tokens=' + process.env.BOUNDED_EXEC_CPU_TOKENS)\" # nextest run";
891
980
 
892
981
  const result = await runBoundedExec(
893
- { command, timeoutMs: 5_000, parallelism: 8, env: { NEXTEST_TEST_THREADS: '64' } },
982
+ { command, timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS, parallelism: 8, env: { NEXTEST_TEST_THREADS: '64' } },
894
983
  workspace.context,
895
984
  createProcessTreeKiller(),
896
985
  null,
@@ -921,7 +1010,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
921
1010
  const result = await runBoundedExec(
922
1011
  {
923
1012
  command: 'node -e "console.log(\'threads=\' + process.env.NEXTEST_TEST_THREADS)" # nextest run',
924
- timeoutMs: 5_000,
1013
+ timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
925
1014
  env: { NEXTEST_TEST_THREADS: '64' },
926
1015
  },
927
1016
  workspace.context,
@@ -946,7 +1035,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
946
1035
  };
947
1036
 
948
1037
  const result = await runBoundedExec(
949
- { command: 'node -e "console.log(\'ran\')"', timeoutMs: 5_000 },
1038
+ { command: 'node -e "console.log(\'ran\')"', timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS },
950
1039
  workspace.context,
951
1040
  createProcessTreeKiller(),
952
1041
  null,
@@ -1019,7 +1108,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
1019
1108
  runBoundedExec(
1020
1109
  {
1021
1110
  command: 'node -e "console.log(\'tokens=\' + process.env.BOUNDED_EXEC_CPU_TOKENS)" # nextest run',
1022
- timeoutMs: 5_000,
1111
+ timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
1023
1112
  parallelism: 8,
1024
1113
  },
1025
1114
  workspace.context,
@@ -1045,7 +1134,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
1045
1134
  runBoundedExec(
1046
1135
  {
1047
1136
  command: 'node -e "console.log(\'threads=\' + process.env.NEXTEST_TEST_THREADS)" # nextest run',
1048
- timeoutMs: 5_000,
1137
+ timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
1049
1138
  parallelism: 2,
1050
1139
  },
1051
1140
  workspace.context,
@@ -1073,7 +1162,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
1073
1162
  };
1074
1163
 
1075
1164
  const result = await runBoundedExec(
1076
- { command: 'node -e "console.log(1)" # nextest run', timeoutMs: 5_000, parallelism: 8 },
1165
+ { command: 'node -e "console.log(1)" # nextest run', timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS, parallelism: 8 },
1077
1166
  workspace.context,
1078
1167
  createProcessTreeKiller(),
1079
1168
  null,
@@ -1108,7 +1197,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
1108
1197
 
1109
1198
  const borrowed = await withPool(3, (pool) =>
1110
1199
  runBoundedExec(
1111
- { command, timeoutMs: 5_000, parallelism: 8 },
1200
+ { command, timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS, parallelism: 8 },
1112
1201
  { ...workspace.context, record: { task, directory: borrowedDir } },
1113
1202
  createProcessTreeKiller(),
1114
1203
  null,
@@ -1117,7 +1206,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
1117
1206
  ),
1118
1207
  );
1119
1208
  const owned = await runBoundedExec(
1120
- { command, timeoutMs: 5_000, parallelism: 8 },
1209
+ { command, timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS, parallelism: 8 },
1121
1210
  { ...workspace.context, record: { task, directory: ownedDir } },
1122
1211
  createProcessTreeKiller(),
1123
1212
  null,
@@ -1155,7 +1244,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
1155
1244
  const running = runBoundedExec(
1156
1245
  {
1157
1246
  command: `sh -c 'echo started > "$0"; read -r line < "$1"' '${started}' '${go}'`,
1158
- timeoutMs: 20_000,
1247
+ timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
1159
1248
  },
1160
1249
  { ...workspace.context, record: { task: { id: 'outlives-its-grant', hash: null }, directory } },
1161
1250
  createProcessTreeKiller(),
@@ -1193,7 +1282,11 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
1193
1282
  const gone = { socket: join(workspace.root, 'gone.sock'), tokens: 3 };
1194
1283
 
1195
1284
  const result = await runBoundedExec(
1196
- { command: 'node -e "console.log(\'ran\')" # nextest run', timeoutMs: 5_000, parallelism: 8 },
1285
+ {
1286
+ command: 'node -e "console.log(\'ran\')" # nextest run',
1287
+ timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
1288
+ parallelism: 8,
1289
+ },
1197
1290
  workspace.context,
1198
1291
  createProcessTreeKiller(),
1199
1292
  null,
@@ -1215,7 +1308,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
1215
1308
  const result = await runBoundedExec(
1216
1309
  {
1217
1310
  command: 'node -e "console.log(\'tokens=\' + process.env.BOUNDED_EXEC_CPU_TOKENS)"',
1218
- timeoutMs: 5_000,
1311
+ timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
1219
1312
  env: { BOUNDED_EXEC_CPU_TOKENS: '977' },
1220
1313
  },
1221
1314
  workspace.context,
@@ -1243,7 +1336,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
1243
1336
  const result = await runBoundedExec(
1244
1337
  {
1245
1338
  command: 'node -e "console.log(\'tokens=\' + process.env.BOUNDED_EXEC_CPU_TOKENS)" # nextest run',
1246
- timeoutMs: 5_000,
1339
+ timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
1247
1340
  parallelism: 8,
1248
1341
  env: { BOUNDED_EXEC_CPU_TOKENS: '977' },
1249
1342
  },
@@ -1280,7 +1373,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
1280
1373
  };
1281
1374
  try {
1282
1375
  const result = await runBoundedExec(
1283
- { command: 'touch ran.marker', timeoutMs: 5_000 },
1376
+ { command: 'touch ran.marker', timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS },
1284
1377
  {
1285
1378
  ...workspace.context,
1286
1379
  record: { task: { id: 'refused', hash: null }, directory: join(workspace.root, 'refused') },
@@ -1318,7 +1411,11 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
1318
1411
  };
1319
1412
  const unbudgeted = await withPool(3, (pool) =>
1320
1413
  runBoundedExec(
1321
- { command: 'node probe.js', timeoutMs: 10_000, env: { ...pool.environment, [CPU_TOKENS_ENV]: '9' } },
1414
+ {
1415
+ command: 'node probe.js',
1416
+ timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
1417
+ env: { ...pool.environment, [CPU_TOKENS_ENV]: '9' },
1418
+ },
1322
1419
  workspace.context,
1323
1420
  createProcessTreeKiller(),
1324
1421
  null,
@@ -1332,7 +1429,11 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
1332
1429
  const granted = await withPool(3, (pool) => {
1333
1430
  injected = inheritedFrom(pool).socket;
1334
1431
  return runBoundedExec(
1335
- { command: 'node probe.js', timeoutMs: 10_000, env: { ...pool.environment, [CPU_TOKENS_ENV]: '9' } },
1432
+ {
1433
+ command: 'node probe.js',
1434
+ timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
1435
+ env: { ...pool.environment, [CPU_TOKENS_ENV]: '9' },
1436
+ },
1336
1437
  workspace.context,
1337
1438
  createProcessTreeKiller(),
1338
1439
  null,
@@ -1364,6 +1465,25 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
1364
1465
  });
1365
1466
  });
1366
1467
 
1468
+ describe('createNoSpaceWatch', () => {
1469
+ it("finds nextest's ENOSPC split across chunks, past colour codes, and reports it once", () => {
1470
+ // nextest's own report of a failed archive extraction into a full RAM lease, from a downstream land gate.
1471
+ const watch = createNoSpaceWatch();
1472
+ expect(watch.scan('stderr', 'Caused by:\n No space left')).toBeNull();
1473
+ expect(watch.scan('stdout', ' on device (os error 28)\n')).toBeNull();
1474
+ expect(watch.scan('stderr', ' on device \u001b[1m(os error 28)\u001b[0m\nCommand exited')).toBe(
1475
+ 'No space left on device (os error 28)',
1476
+ );
1477
+ expect(watch.scan('stderr', 'ENOSPC: no space left on device, write\n')).toBeNull();
1478
+ });
1479
+
1480
+ it("finds Bun's errno name, and not a word that merely contains it", () => {
1481
+ const watch = createNoSpaceWatch();
1482
+ expect(watch.scan('stdout', 'XENOSPCY is not an errno\n')).toBeNull();
1483
+ expect(watch.scan('stdout', ' code: "ENOSPC"\n')).toBe('code: "ENOSPC"');
1484
+ });
1485
+ });
1486
+
1367
1487
  interface WorkspaceFixture {
1368
1488
  root: string;
1369
1489
  context: BoundedExecContext;
@@ -1424,17 +1544,18 @@ async function armedPid(path: string, pgid: number, subject: string): Promise<nu
1424
1544
  }
1425
1545
 
1426
1546
  /**
1427
- * A command that ignores SIGTERM, a disposition its `sleep` inherits, and `armed`, which resolves once it does. A
1428
- * killer that awaits `armed` before the graceful signal sends it to a command that cannot die of it: a bound that fires
1429
- * before the command's shell has even run `trap` (a 250 ms timer lost that race in a loaded gate) ends it on the
1430
- * default disposition, and the escalation under test never runs. The command's FIFO write waits for the reader, so
1431
- * the announcement can neither precede the trap nor be missed.
1547
+ * A command that ignores SIGTERM, a disposition its `tail` inherits, and never exits on its own, so only the escalation
1548
+ * can end it; and `armed`, which resolves once it ignores SIGTERM. A killer that awaits `armed` before the graceful
1549
+ * signal sends it to a command that cannot die of it: a bound that fires before the command's shell has even run
1550
+ * `trap` (a 250 ms timer lost that race in a loaded gate) ends it on the default disposition, and the escalation under
1551
+ * test never runs. The command's FIFO write waits for the reader, so the announcement can neither precede the trap nor
1552
+ * be missed.
1432
1553
  */
1433
1554
  function ignoresTerm(root: string): { command: string; armed: () => Promise<void> } {
1434
1555
  const fifo = join(root, 'armed.fifo');
1435
1556
  expect(spawnSync('mkfifo', [fifo]).status).toBe(0);
1436
1557
  return {
1437
- command: `sh -c 'trap "" TERM; echo armed > "$0"; exec sleep 5' '${fifo}'`,
1558
+ command: `sh -c 'trap "" TERM; echo armed > "$0"; exec tail -f /dev/null' '${fifo}'`,
1438
1559
  armed: async () => {
1439
1560
  expect(await readToEnd(fifo)).toBe('armed\n');
1440
1561
  },
@@ -1459,11 +1580,12 @@ function leaseAt(directory: string): RamTempLease {
1459
1580
  return { directory, mountpoint: dirname(directory), capacityBytes: 1024 * 1024 * 1024 };
1460
1581
  }
1461
1582
 
1462
- /** A temp volume whose answers are fixed, recording which leases were reaped and released, in order. */
1583
+ /** A temp volume whose answers are fixed, recording which leases were reaped and released, and when space was read. */
1463
1584
  function scriptedVolume(
1464
1585
  acquisition: Result<RamTempAcquisition, RamTempError>,
1465
1586
  full: RamTempError | null = null,
1466
1587
  leftovers: string[] = [],
1588
+ space: TempVolumeSpace | null = null,
1467
1589
  ): TempVolume & { released: string[]; order: string[] } {
1468
1590
  const released: string[] = [];
1469
1591
  const order: string[] = [];
@@ -1481,6 +1603,10 @@ function scriptedVolume(
1481
1603
  return Promise.resolve({ ok: true, value: undefined });
1482
1604
  },
1483
1605
  fullness: () => Promise.resolve(full),
1606
+ space() {
1607
+ order.push('space');
1608
+ return Promise.resolve(space);
1609
+ },
1484
1610
  };
1485
1611
  }
1486
1612