@smoothbricks/nx-plugin 0.4.28 → 0.4.30

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (158) hide show
  1. package/README.md +277 -89
  2. package/dist/bin/smoo-nx-test-clocks.d.ts +3 -0
  3. package/dist/bin/smoo-nx-test-clocks.d.ts.map +1 -0
  4. package/dist/bin/smoo-nx-test-clocks.js +28 -0
  5. package/dist/bounded-test-policy.d.ts +8 -8
  6. package/dist/bounded-test-policy.d.ts.map +1 -1
  7. package/dist/bounded-test-policy.js +18 -30
  8. package/dist/cargo-closure-input.d.ts +23 -0
  9. package/dist/cargo-closure-input.d.ts.map +1 -1
  10. package/dist/cargo-closure-input.js +41 -9
  11. package/dist/cargo-source-hash.d.ts.map +1 -1
  12. package/dist/cargo-source-hash.js +42 -19
  13. package/dist/ensure-built-receipt.d.ts +40 -6
  14. package/dist/ensure-built-receipt.d.ts.map +1 -1
  15. package/dist/ensure-built-receipt.js +297 -159
  16. package/dist/ensure-built-witness.d.ts +65 -4
  17. package/dist/ensure-built-witness.d.ts.map +1 -1
  18. package/dist/ensure-built-witness.js +2237 -23
  19. package/dist/ensure-built.d.ts +99 -1
  20. package/dist/ensure-built.d.ts.map +1 -1
  21. package/dist/ensure-built.js +45 -56
  22. package/dist/executors/bounded-exec/cpu-pool.d.ts +22 -7
  23. package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -1
  24. package/dist/executors/bounded-exec/cpu-pool.js +51 -22
  25. package/dist/executors/bounded-exec/executor.d.ts +12 -3
  26. package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
  27. package/dist/executors/bounded-exec/executor.js +174 -51
  28. package/dist/executors/bounded-exec/no-space.d.ts +8 -0
  29. package/dist/executors/bounded-exec/no-space.d.ts.map +1 -0
  30. package/dist/executors/bounded-exec/no-space.js +31 -0
  31. package/dist/executors/bounded-exec/ram-temp.d.ts +20 -11
  32. package/dist/executors/bounded-exec/ram-temp.d.ts.map +1 -1
  33. package/dist/executors/bounded-exec/ram-temp.js +42 -17
  34. package/dist/executors/bounded-exec/verdict.d.ts +34 -4
  35. package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
  36. package/dist/executors/bounded-exec/verdict.js +54 -3
  37. package/dist/generators/bun-test-tracing/generator.js +2 -2
  38. package/dist/graph-input-contract.d.ts +102 -0
  39. package/dist/graph-input-contract.d.ts.map +1 -0
  40. package/dist/graph-input-contract.js +637 -0
  41. package/dist/index.d.ts +12 -5
  42. package/dist/index.d.ts.map +1 -1
  43. package/dist/index.js +17 -3
  44. package/dist/managed-files/ci-workflow.d.ts.map +1 -1
  45. package/dist/managed-files/ci-workflow.js +5 -2
  46. package/dist/managed-files/publish-workflow.d.ts +11 -1
  47. package/dist/managed-files/publish-workflow.d.ts.map +1 -1
  48. package/dist/owned-installation.d.ts +14 -0
  49. package/dist/owned-installation.d.ts.map +1 -0
  50. package/dist/owned-installation.js +324 -0
  51. package/dist/producer-receipts.d.ts +19 -0
  52. package/dist/producer-receipts.d.ts.map +1 -0
  53. package/dist/producer-receipts.js +926 -0
  54. package/dist/test-backstop.d.ts +9 -0
  55. package/dist/test-backstop.d.ts.map +1 -0
  56. package/dist/test-backstop.js +10 -0
  57. package/dist/test-clock-policy.d.ts +30 -0
  58. package/dist/test-clock-policy.d.ts.map +1 -0
  59. package/dist/test-clock-policy.js +686 -0
  60. package/dist/testing.d.ts +8 -0
  61. package/dist/testing.d.ts.map +1 -1
  62. package/dist/testing.js +66 -1
  63. package/managed/raw/patches/nx@23.2.1.patch +328 -50
  64. package/managed/raw/tooling/direnv/developer-links.ts +148 -29
  65. package/managed/raw/tooling/direnv/devenv.smoo.nix +7 -7
  66. package/managed/raw/tooling/direnv/github-actions-bootstrap.sh +14 -5
  67. package/managed/raw/tooling/direnv/linked-snapshots.ts +206 -40
  68. package/managed/raw/tooling/direnv/repo-path +15 -2
  69. package/managed/raw/tooling/direnv/setup-environment.ts +179 -115
  70. package/managed/templates/github/actions/cache-node-modules/action.yml +9 -0
  71. package/managed/templates/github/actions/setup-devenv/action.yml +63 -7
  72. package/nextest.toml +1 -1
  73. package/package.json +25 -9
  74. package/src/__tests__/cargo-closure-fixture.ts +113 -0
  75. package/src/__tests__/counted-cargo.ts +8 -25
  76. package/src/__tests__/directory-change.ts +27 -0
  77. package/src/__tests__/fixture-nx-env.ts +7 -0
  78. package/src/__tests__/owned-installation.ts +14 -0
  79. package/src/__tests__/past-tick.ts +12 -0
  80. package/src/bin/smoo-nx-bound-failures.test.ts +1 -1
  81. package/src/bin/smoo-nx-test-clocks.ts +29 -0
  82. package/src/bounded-test-policy.test.ts +36 -22
  83. package/src/bounded-test-policy.ts +18 -32
  84. package/src/bun-bootstrap.test.ts +1 -1
  85. package/src/cargo-closure-input.property.test.ts +50 -0
  86. package/src/cargo-closure-input.test.ts +73 -121
  87. package/src/cargo-closure-input.ts +47 -8
  88. package/src/cargo-closure-key.test.ts +239 -0
  89. package/src/cargo-resolve-cache.test.ts +168 -31
  90. package/src/cargo-source-hash.test.ts +78 -3
  91. package/src/cargo-source-hash.ts +48 -30
  92. package/src/cross-check-policy.test.ts +2 -2
  93. package/src/ensure-built-receipt.test.ts +111 -16
  94. package/src/ensure-built-receipt.ts +294 -149
  95. package/src/ensure-built-witness.test.ts +306 -13
  96. package/src/ensure-built-witness.ts +255 -24
  97. package/src/ensure-built.test.ts +228 -102
  98. package/src/ensure-built.ts +127 -66
  99. package/src/executors/bounded-exec/cpu-pool.test.ts +134 -32
  100. package/src/executors/bounded-exec/cpu-pool.ts +64 -20
  101. package/src/executors/bounded-exec/cpu-tokens.test.ts +2 -2
  102. package/src/executors/bounded-exec/disk-lease.test.ts +47 -13
  103. package/src/executors/bounded-exec/executor-host.test.ts +74 -52
  104. package/src/executors/bounded-exec/executor.test.ts +343 -97
  105. package/src/executors/bounded-exec/executor.ts +192 -56
  106. package/src/executors/bounded-exec/hang-diagnostics.test.ts +14 -8
  107. package/src/executors/bounded-exec/no-space.ts +40 -0
  108. package/src/executors/bounded-exec/progress.test.ts +1 -1
  109. package/src/executors/bounded-exec/ram-temp-host.test.ts +51 -21
  110. package/src/executors/bounded-exec/ram-temp.test.ts +92 -16
  111. package/src/executors/bounded-exec/ram-temp.ts +49 -20
  112. package/src/executors/bounded-exec/verdict.test.ts +127 -22
  113. package/src/executors/bounded-exec/verdict.ts +78 -3
  114. package/src/generators/bounded-test-targets/generator.test.ts +1 -3
  115. package/src/generators/bun-test-tracing/generator.test.ts +2 -2
  116. package/src/generators/bun-test-tracing/generator.ts +2 -2
  117. package/src/graph-input-contract.test.ts +379 -0
  118. package/src/graph-input-contract.ts +627 -0
  119. package/src/index.test.ts +3 -3
  120. package/src/index.ts +24 -3
  121. package/src/linked-snapshots.nx.test.ts +112 -30
  122. package/src/managed-files/ci-workflow.ts +5 -2
  123. package/src/managed-files/publish-workflow.ts +12 -1
  124. package/src/managed-files/repo-path.test.ts +44 -1
  125. package/src/managed-files/setup-devenv-free-disk.test.ts +95 -0
  126. package/src/nextest-code-bound.test.ts +23 -17
  127. package/src/nextest-extraction.test.ts +114 -4
  128. package/src/nextest-partitions.test.ts +7 -1
  129. package/src/nx-cache-restore-mtime.test.ts +2 -2
  130. package/src/nx-daemon-claim.test.ts +10 -8
  131. package/src/nx-daemon-idle-cpu.test.ts +0 -1
  132. package/src/nx-daemon-idle-policy.test.ts +61 -1
  133. package/src/nx-daemon-ignore-files.test.ts +2 -2
  134. package/src/nx-daemon-validity.test.ts +36 -22
  135. package/src/nx-dot-input-policy.test.ts +5 -5
  136. package/src/nx-files-archive-racy.test.ts +1 -1
  137. package/src/nx-incomplete-run-summary.test.ts +1 -1
  138. package/src/nx-plugin-load-errors.test.ts +2 -2
  139. package/src/nx-project-graph-errors.test.ts +70 -0
  140. package/src/nx-skipped-task-exit.test.ts +2 -2
  141. package/src/nx-store-resolution.test.ts +7 -25
  142. package/src/nx-task-output-lease.test.ts +3 -3
  143. package/src/nx-use-local.test.ts +7 -5
  144. package/src/owned-installation.ts +174 -0
  145. package/src/producer-receipts.test.ts +164 -0
  146. package/src/producer-receipts.ts +636 -0
  147. package/src/runtime-input.test.ts +1 -1
  148. package/src/serial-group-wiring.test.ts +4 -4
  149. package/src/task-results.test.ts +18 -16
  150. package/src/test-backstop.ts +11 -0
  151. package/src/test-clock-policy.test.ts +338 -0
  152. package/src/test-clock-policy.ts +708 -0
  153. package/src/test-timings.test.ts +2 -2
  154. package/src/testing.test.ts +5 -5
  155. package/src/testing.ts +67 -1
  156. package/src/typecheck-test-policy.test.ts +1 -1
  157. package/src/typescript-resolution.test.ts +76 -0
  158. package/test-timings.json +8 -0
@@ -1,23 +1,27 @@
1
- import { afterEach, describe, expect, it } from 'bun:test';
1
+ import { afterEach, describe, expect, it, jest } from 'bun:test';
2
2
  import { spawnSync } from 'node:child_process';
3
3
  import { access, mkdir, mkdtemp, open, readFile, rm, writeFile } from 'node:fs/promises';
4
4
  import { createServer } from 'node:net';
5
- import { availableParallelism, tmpdir } from 'node:os';
5
+ import { tmpdir } from 'node:os';
6
6
  import { dirname, join } from 'node:path';
7
7
  import { onProgressClock } from '../../__tests__/progress-clock.js';
8
+ import { BOUNDED_TEST_IDLE_TIMEOUT_MS } from '../../bounded-test-policy.js';
8
9
  import { ownedSocketDirectory } from '../../testing.js';
9
10
  import { CPU_POOL_ENV, type CpuPool, createCpuPool, type InheritedCpuPool, readInheritedCpuPool } from './cpu-pool.js';
10
11
  import { CPU_TOKENS_ENV, type CpuBudget, parseInheritedTokens, type TokenGrant } from './cpu-tokens.js';
11
12
  import {
12
13
  type BoundedExecContext,
14
+ type BoundedExecResult,
13
15
  createProcessTreeKiller,
14
16
  type ProcessTreeKiller,
17
+ resolveCowshedExecutable,
15
18
  runBoundedExec,
16
19
  type TempVolume,
17
20
  } from './executor.js';
18
21
  import type { HangDiagnostics } from './hang-diagnostics.js';
22
+ import { createNoSpaceWatch } from './no-space.js';
19
23
  import type { RamTempAcquisition, RamTempError, RamTempLease, Result } from './ram-temp.js';
20
- import { VERDICT_FILE } from './verdict.js';
24
+ import { readRecord, type TempVolumeSpace, VERDICT_FILE } from './verdict.js';
21
25
 
22
26
  const workspaces: string[] = [];
23
27
  const ownedProcessGroups = new Set<number>();
@@ -44,7 +48,10 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
44
48
  try {
45
49
  process.kill(-pid, 'SIGKILL');
46
50
  } catch (cause) {
47
- if (!(cause instanceof Error && 'code' in cause && cause.code === 'ESRCH')) {
51
+ const code = cause instanceof Error && 'code' in cause ? cause.code : undefined;
52
+ // Darwin answers EPERM, not ESRCH, for a group whose only members are unreaped zombies (a killed
53
+ // descendant waiting for launchd to reap it): such a group has nothing left to kill.
54
+ if (code !== 'ESRCH' && !(code === 'EPERM' && onlyZombiesIn(pid))) {
48
55
  throw new Error(`cleanup owned readiness process group pgid=${pid} failed`, { cause });
49
56
  }
50
57
  } finally {
@@ -61,12 +68,14 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
61
68
  'console.log(process.cwd()); console.log(process.env.BOUNDED_EXEC_VALUE);\n',
62
69
  );
63
70
 
71
+ // bounded-exec requires a ceiling; at the leg's backstop it cannot end a run whose `node` is slow to start
72
+ // under load.
64
73
  const result = await runBoundedExec(
65
74
  {
66
75
  command: 'node print.js',
67
76
  cwd: 'package',
68
77
  env: { BOUNDED_EXEC_VALUE: 'from-env' },
69
- timeoutMs: 5_000,
78
+ timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
70
79
  },
71
80
  workspace.context,
72
81
  createProcessTreeKiller(),
@@ -83,7 +92,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
83
92
  const workspace = await createWorkspace();
84
93
 
85
94
  const result = await runBoundedExec(
86
- { command: 'node -e "process.exit(7)"', timeoutMs: 5_000 },
95
+ { command: 'node -e "process.exit(7)"', timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS },
87
96
  workspace.context,
88
97
  createProcessTreeKiller(),
89
98
  null,
@@ -102,7 +111,10 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
102
111
  process.stderr.write = captureWrite(stderr);
103
112
 
104
113
  const result = await runBoundedExec(
105
- { command: "node -e \"console.log('out-value'); console.error('err-value')\"", timeoutMs: 5_000 },
114
+ {
115
+ command: "node -e \"console.log('out-value'); console.error('err-value')\"",
116
+ timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
117
+ },
106
118
  workspace.context,
107
119
  createProcessTreeKiller(),
108
120
  null,
@@ -119,8 +131,9 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
119
131
  it('fails on timeout and reports bounded execution details', async () => {
120
132
  const workspace = await createWorkspace();
121
133
 
134
+ // The command never exits, so only the ceiling under test can end it.
122
135
  const result = await runBoundedExec(
123
- { command: 'node -e "setTimeout(() => {}, 5000)"', timeoutMs: 50, killAfterMs: 0 },
136
+ { command: 'node -e "process.stdin.resume()"', timeoutMs: 50, killAfterMs: 0 },
124
137
  workspace.context,
125
138
  createProcessTreeKiller(),
126
139
  null,
@@ -136,8 +149,9 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
136
149
  it('kills a silent command on the progress bound, with no ceiling at all', async () => {
137
150
  const workspace = await createWorkspace();
138
151
 
152
+ // The command never exits and never speaks, so only the progress bound under test can end it.
139
153
  const result = await runBoundedExec(
140
- { command: 'node -e "setTimeout(() => {}, 30000)"', idleTimeoutMs: 100, killAfterMs: 0 },
154
+ { command: 'node -e "process.stdin.resume()"', idleTimeoutMs: 100, killAfterMs: 0 },
141
155
  workspace.context,
142
156
  createProcessTreeKiller(),
143
157
  null,
@@ -186,6 +200,56 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
186
200
  expect(result.terminalOutput).not.toContain('fake tests::quick (cwd=');
187
201
  });
188
202
 
203
+ it("stops the idle clock while a descendant's work is queued for host CPU tokens, and runs it on after", async () => {
204
+ const workspace = await createWorkspace();
205
+ // The runner finishes a test, then says through its pool that work it started is queued for host tokens, ends that
206
+ // wait and goes silent. While the wait is held the clock moves well past the bound and nothing fires; once the wait
207
+ // ends the same move ends the run.
208
+ await workspace.write(
209
+ 'queued-runner.js',
210
+ [
211
+ "const { connect } = require('node:net');",
212
+ "console.error(' PASS [ 0.010s] (1/2) fake tests::quick');",
213
+ `const socket = connect(process.env.${CPU_POOL_ENV});`,
214
+ `socket.on('connect', () => socket.write('{"op":"host-wait"}\\n'));`,
215
+ "socket.once('data', (reply) => { console.log('host wait ' + reply.toString().trim()); socket.end(); });",
216
+ "socket.on('close', () => setInterval(() => {}, 60_000));",
217
+ '',
218
+ ].join('\n'),
219
+ );
220
+ const budget: CpuBudget = {
221
+ async take() {
222
+ return heldGrant(1, []);
223
+ },
224
+ };
225
+
226
+ const { result } = await onProgressClock(
227
+ (chunk) => chunk.includes('bounded-exec: idle clock'),
228
+ 1_000,
229
+ () =>
230
+ runBoundedExec(
231
+ { command: 'node queued-runner.js', idleTimeoutMs: 400, killAfterMs: 0 },
232
+ workspace.context,
233
+ createProcessTreeKiller(),
234
+ null,
235
+ budget,
236
+ ),
237
+ );
238
+
239
+ const output = result.terminalOutput;
240
+ expect(result.success).toBe(false);
241
+ expect(output).toContain('host wait {"ok":true,"wait":"held"}');
242
+ const stopped = output.indexOf(
243
+ "bounded-exec: idle clock stopped: a descendant's work is queued for host CPU tokens",
244
+ );
245
+ const restarted = output.indexOf('bounded-exec: idle clock restarted');
246
+ const expired = output.indexOf('Command made no progress: no test finished for 400ms (idleTimeoutMs=400)');
247
+ // The 1000 ms move while the wait was held fired nothing; the one after it ended the run.
248
+ expect(stopped).toBeGreaterThan(-1);
249
+ expect(restarted).toBeGreaterThan(stopped);
250
+ expect(expired).toBeGreaterThan(restarted);
251
+ });
252
+
189
253
  it('emits a completed pre-expiry snapshot or an explicit incomplete capture without waiting for it', async () => {
190
254
  const workspace = await createWorkspace();
191
255
  // Child output drives the fake executor clock; the heartbeat period is not a timing oracle.
@@ -381,7 +445,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
381
445
  await workspace.write('runaway.js', ["setInterval(() => console.log('still here'), 10);", ''].join('\n'));
382
446
 
383
447
  const result = await runBoundedExec(
384
- { command: 'node runaway.js', timeoutMs: 300, idleTimeoutMs: 30_000, killAfterMs: 0 },
448
+ { command: 'node runaway.js', timeoutMs: 300, idleTimeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS, killAfterMs: 0 },
385
449
  workspace.context,
386
450
  createProcessTreeKiller(),
387
451
  null,
@@ -396,17 +460,47 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
396
460
 
397
461
  it('imposes no progress bound when idleTimeoutMs is omitted', async () => {
398
462
  const workspace = await createWorkspace();
399
-
400
- const result = await runBoundedExec(
401
- { command: 'node -e "setTimeout(() => console.log(\'late\'), 400)"', timeoutMs: 30_000 },
402
- workspace.context,
403
- createProcessTreeKiller(),
404
- null,
405
- null,
463
+ const go = join(workspace.root, 'go.fifo');
464
+ expect(spawnSync('mkfifo', [go]).status).toBe(0);
465
+ // Says `started`, then goes silent until the test writes `go.fifo`.
466
+ await workspace.write(
467
+ 'silent.js',
468
+ [
469
+ "const fs = require('node:fs');",
470
+ "console.log('started');",
471
+ "fs.readFileSync('go.fifo');",
472
+ "console.log('late');",
473
+ '',
474
+ ].join('\n'),
406
475
  );
476
+ const { promise: started, resolve: sawStarted } = Promise.withResolvers<void>();
477
+ process.stdout.write = captureWrite([], (text) => {
478
+ if (text.includes('started')) sawStarted();
479
+ });
480
+ // Every bound is armed as the command spawns, before it can say `started`. The clock then moves to just short of
481
+ // the ceiling while the command is silent: any progress bound would have fired there, and the ceiling has not.
482
+ // Only then is the command released, so the run ends by its own exit.
483
+ jest.useFakeTimers();
484
+ let result: BoundedExecResult;
485
+ try {
486
+ const run = runBoundedExec(
487
+ { command: 'node silent.js', timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS },
488
+ workspace.context,
489
+ createProcessTreeKiller(),
490
+ null,
491
+ null,
492
+ );
493
+ await started;
494
+ jest.advanceTimersByTime(BOUNDED_TEST_IDLE_TIMEOUT_MS - 1);
495
+ await writeFile(go, 'go\n');
496
+ result = await run;
497
+ } finally {
498
+ jest.useRealTimers();
499
+ }
407
500
 
408
501
  expect(result.success).toBe(true);
409
502
  expect(result.terminalOutput).toContain('late');
503
+ expect(result.terminalOutput).not.toContain('made no progress');
410
504
  });
411
505
 
412
506
  it('uses graceful timeout termination before force-killing on POSIX', async () => {
@@ -427,7 +521,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
427
521
  ' process.exit(0);',
428
522
  '});',
429
523
  "fs.writeFileSync('cooperative-ready.fifo', 'ready pid=' + process.pid + '\\n');",
430
- 'setTimeout(() => {}, 5000);',
524
+ 'process.stdin.resume();',
431
525
  '',
432
526
  ].join('\n'),
433
527
  );
@@ -437,8 +531,10 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
437
531
  expect(handlerPid, `handler readiness must name the command leader pid=${pid}`).toBe(pid);
438
532
  });
439
533
 
534
+ // The ceiling is the subject; the grace after the graceful signal is the leg's backstop, so the command ends by its
535
+ // handler alone however slowly a loaded host runs it.
440
536
  const result = await runBoundedExec(
441
- { command: 'exec node cooperative.js', timeoutMs: 50, killAfterMs: 500 },
537
+ { command: 'exec node cooperative.js', timeoutMs: 50, killAfterMs: BOUNDED_TEST_IDLE_TIMEOUT_MS },
442
538
  workspace.context,
443
539
  termination.killer,
444
540
  null,
@@ -498,15 +594,18 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
498
594
  leaked = pid;
499
595
  },
500
596
  };
501
- const started = Date.now();
502
597
  const result = await runBoundedExec(
503
- { command: 'node -e "setTimeout(() => {}, 30000)"', timeoutMs: 50, killAfterMs: 10 },
598
+ { command: 'node -e "process.stdin.resume()"', timeoutMs: 50, killAfterMs: 10 },
504
599
  workspace.context,
505
600
  killer,
506
601
  null,
507
602
  null,
508
603
  );
509
- expect(Date.now() - started).toBeLessThan(5_000);
604
+ // The command never exits and the killer delivers nothing, so the run returned by giving up on the tree after its
605
+ // force-kill, not by the command ending: the command is still alive here.
606
+ if (leaked === undefined) throw new Error('the killer was never asked to kill the command');
607
+ const pid = leaked;
608
+ expect(() => process.kill(pid, 0)).not.toThrow();
510
609
  expect(result.success).toBe(false);
511
610
  expect(result.terminalOutput).toContain('Force-killing timed out command');
512
611
  if (leaked !== undefined) {
@@ -531,7 +630,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
531
630
  command: 'node -e "console.log(process.argv.slice(1).join(\',\'))"',
532
631
  args: ['first'],
533
632
  __unparsed__: ['second'],
534
- timeoutMs: 5_000,
633
+ timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
535
634
  },
536
635
  workspace.context,
537
636
  createProcessTreeKiller(),
@@ -544,7 +643,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
544
643
  args: ['first'],
545
644
  __unparsed__: ['second'],
546
645
  forwardAllArgs: false,
547
- timeoutMs: 5_000,
646
+ timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
548
647
  },
549
648
  workspace.context,
550
649
  createProcessTreeKiller(),
@@ -565,7 +664,6 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
565
664
  }
566
665
 
567
666
  const workspace = await createWorkspace();
568
- const marker = join(workspace.root, 'marker.txt');
569
667
  const ready = join(workspace.root, 'descendant-ready.fifo');
570
668
  const lifetime = join(workspace.root, 'descendant-lifetime.fifo');
571
669
  expect(spawnSync('mkfifo', [ready, lifetime]).status).toBe(0);
@@ -575,7 +673,8 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
575
673
  "const fs = require('node:fs');",
576
674
  "fs.openSync('descendant-lifetime.fifo', 'w');",
577
675
  "fs.writeFileSync('descendant-ready.fifo', 'ready pid=' + process.pid + '\\n');",
578
- "setTimeout(() => fs.writeFileSync(process.argv[2], 'alive'), 700);",
676
+ // Its stdin is /dev/null, so it holds itself: a wait no event ever ends, until a signal does.
677
+ 'Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0);',
579
678
  '',
580
679
  ].join('\n'),
581
680
  );
@@ -583,12 +682,12 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
583
682
  'spawn-child.js',
584
683
  [
585
684
  "import { spawn } from 'node:child_process';",
586
- "spawn(process.execPath, ['group-child.js', process.argv[2]], { stdio: 'ignore' });",
587
- 'setTimeout(() => {}, 5000);',
685
+ "spawn(process.execPath, ['group-child.js'], { stdio: 'ignore' });",
686
+ 'process.stdin.resume();',
588
687
  '',
589
688
  ].join('\n'),
590
689
  );
591
- // Only the descendant owns the writer; EOF observes its actual terminal closure, not a time horizon.
690
+ // Only the descendant owns the writer and it never exits on its own: EOF observes its kill, not a time horizon.
592
691
  const closed = readToEnd(lifetime);
593
692
  let descendantPid: number | undefined;
594
693
  const termination = armedKiller('process-group descendant', async (pid) => {
@@ -597,7 +696,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
597
696
  });
598
697
 
599
698
  const result = await runBoundedExec(
600
- { command: `exec node spawn-child.js ${marker}`, timeoutMs: 50, killAfterMs: 50 },
699
+ { command: 'exec node spawn-child.js', timeoutMs: 50, killAfterMs: 50 },
601
700
  workspace.context,
602
701
  termination.killer,
603
702
  null,
@@ -607,7 +706,6 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
607
706
  await termination.armed();
608
707
  expect(await closed, `ready descendant pid=${descendantPid} must close its lifetime FIFO on exit`).toBe('');
609
708
  expect(result.success).toBe(false);
610
- expect(await exists(marker)).toBe(false);
611
709
  });
612
710
 
613
711
  it('force-kills what outlives a timed-out command in its process group', async () => {
@@ -620,27 +718,26 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
620
718
  const ready = join(workspace.root, 'straggler-ready.fifo');
621
719
  expect(spawnSync('mkfifo', [fifo, ready]).status).toBe(0);
622
720
  // The leader exits on SIGTERM. The subshell beside it ignores SIGTERM, a disposition its
623
- // `sleep` inherits, and announces itself only once it holds the FIFO: from then on the
721
+ // `tail` inherits, and announces itself only once it holds the FIFO: from then on the
624
722
  // graceful signal cannot end it. A process forked while that signal goes out is the same
625
723
  // case, without the trap.
626
724
  await workspace.write(
627
725
  'straggle.sh',
628
726
  [
629
- '( trap "" TERM; exec 3>"$1"; echo ready; echo armed >"$2"; sleep 10; echo survived >&3 ) | {',
630
- ' read -r line; echo "$line"; exec sleep 30',
727
+ '( trap "" TERM; exec 3>"$1"; echo ready; echo armed >"$2"; exec tail -f /dev/null ) | {',
728
+ ' read -r line; echo "$line"; exec tail -f /dev/null',
631
729
  '}',
632
730
  '',
633
731
  ].join('\n'),
634
732
  );
635
- // The straggler is the FIFO's only writer, so the read ends when it is gone: with nothing
636
- // written if it was killed, with "survived" if it lived to finish its sleep.
733
+ // The straggler is the FIFO's only writer and never ends on its own, so the read ends only when it is killed.
637
734
  const straggler = readToEnd(fifo);
638
735
  const termination = armedKiller('TERM-ignoring FIFO straggler', async (pid) => {
639
736
  expect(await readToEnd(ready), `straggler readiness pgid=${pid} must follow trap and FIFO open`).toBe('armed\n');
640
737
  });
641
738
 
642
739
  const result = await runBoundedExec(
643
- { command: `sh straggle.sh ${fifo} ${ready}`, timeoutMs: 5_000, killAfterMs: 30_000 },
740
+ { command: `sh straggle.sh ${fifo} ${ready}`, timeoutMs: 5_000, killAfterMs: BOUNDED_TEST_IDLE_TIMEOUT_MS },
644
741
  workspace.context,
645
742
  termination.killer,
646
743
  null,
@@ -652,7 +749,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
652
749
  expect(await straggler).toBe('');
653
750
  expect(result.success).toBe(false);
654
751
  expect(result.terminalOutput).toContain('Force-killing what the timed-out command left in its process group');
655
- }, 60_000);
752
+ });
656
753
 
657
754
  it('runs the command with its RAM lease as TMPDIR, names held dead leases, and ends the lease after it exits', async () => {
658
755
  const workspace = await createWorkspace();
@@ -665,7 +762,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
665
762
  });
666
763
 
667
764
  const result = await runBoundedExec(
668
- { command: 'node -e "console.log(process.env.TMPDIR)"', timeoutMs: 5_000 },
765
+ { command: 'node -e "console.log(process.env.TMPDIR)"', timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS },
669
766
  workspace.context,
670
767
  createProcessTreeKiller(),
671
768
  volume,
@@ -692,7 +789,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
692
789
  );
693
790
 
694
791
  const result = await runBoundedExec(
695
- { command: 'node -e "0"', timeoutMs: 5_000 },
792
+ { command: 'node -e "0"', timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS },
696
793
  workspace.context,
697
794
  createProcessTreeKiller(),
698
795
  volume,
@@ -714,7 +811,11 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
714
811
  const volume = scriptedVolume({ ok: true, value: { kind: 'sandboxed', detail: 'lock: EPERM' } });
715
812
 
716
813
  const result = await runBoundedExec(
717
- { command: `node -e 'console.log("tmp=" + process.env.TMPDIR)'`, timeoutMs: 5_000, env: { TMPDIR: '/shed/tmp' } },
814
+ {
815
+ command: `node -e 'console.log("tmp=" + process.env.TMPDIR)'`,
816
+ timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
817
+ env: { TMPDIR: '/shed/tmp' },
818
+ },
718
819
  workspace.context,
719
820
  createProcessTreeKiller(),
720
821
  volume,
@@ -736,7 +837,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
736
837
  });
737
838
 
738
839
  const result = await runBoundedExec(
739
- { command: `touch ${marker}`, timeoutMs: 5_000 },
840
+ { command: `touch ${marker}`, timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS },
740
841
  workspace.context,
741
842
  createProcessTreeKiller(),
742
843
  volume,
@@ -758,7 +859,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
758
859
  );
759
860
 
760
861
  const result = await runBoundedExec(
761
- { command: 'node -e "process.exit(1)"', timeoutMs: 5_000 },
862
+ { command: 'node -e "process.exit(1)"', timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS },
762
863
  workspace.context,
763
864
  createProcessTreeKiller(),
764
865
  volume,
@@ -770,6 +871,48 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
770
871
  expect(volume.released).toEqual([directory]);
771
872
  });
772
873
 
874
+ it('names ENOSPC, its writer and the volume as it stood when the output said so, not after the command exited', async () => {
875
+ // From a downstream land gate: a compiler test shard printed ENOSPC for a 1 KB write into its lease and failed two.
876
+ // By its exit the volume had space again (`fullness` answers null here), so a probe after exit named nothing and
877
+ // the run read as two failed tests.
878
+ const workspace = await createWorkspace();
879
+ const directory = join(workspace.root, 'lease');
880
+ await mkdir(directory);
881
+ const lease = leaseAt(directory);
882
+ const space = { lease: directory, mountpoint: workspace.root, capacityBytes: lease.capacityBytes, freeBytes: 4096 };
883
+ const volume = scriptedVolume(
884
+ { ok: true, value: { kind: 'leased', lease, held: [], reaped: [] } },
885
+ null,
886
+ [],
887
+ space,
888
+ );
889
+ const record = { task: { id: 'compiler:test-10', hash: 'h' }, directory: join(workspace.root, 'record') };
890
+ const line = 'ENOSPC: no space left on device, write';
891
+
892
+ const result = await runBoundedExec(
893
+ { command: `node -e "console.error('${line}'); process.exit(1)"`, timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS },
894
+ { ...workspace.context, record },
895
+ createProcessTreeKiller(),
896
+ volume,
897
+ null,
898
+ );
899
+
900
+ expect(result.success).toBe(false);
901
+ expect(result.terminalOutput).toContain(
902
+ `compiler:test-10: ENOSPC (no space left on device): the command printed "${line}"; its TMPDIR ` +
903
+ `${directory} is on the RAM temp volume ${workspace.root}, which had 0.0 MiB free of 1024.0 MiB when that ` +
904
+ 'line arrived\n',
905
+ );
906
+ expect(result.terminalOutput).not.toContain('is full:');
907
+ expect(await readRecord(record.directory)).toEqual({
908
+ task: record.task.id,
909
+ hash: 'h',
910
+ verdict: { outcome: 'no-space', exitCode: 1, tests: [], line, volume: space },
911
+ });
912
+ // Read while the lease was still the task's, before it was reaped and released.
913
+ expect(volume.order).toEqual(['space', `reap ${directory}`, `release ${directory}`]);
914
+ });
915
+
773
916
  it('keeps inherited nextest and Cargo ceilings under oversized grants', async () => {
774
917
  const previousNextest = process.env.NEXTEST_TEST_THREADS;
775
918
  const previousCargo = process.env.CARGO_BUILD_JOBS;
@@ -796,7 +939,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
796
939
  const result = await runBoundedExec(
797
940
  {
798
941
  command,
799
- timeoutMs: 5_000,
942
+ timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
800
943
  parallelism: 18,
801
944
  env: { NEXTEST_TEST_THREADS: '64', CARGO_BUILD_JOBS: '64', RUST_TEST_THREADS: '64' },
802
945
  },
@@ -836,7 +979,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
836
979
  "node -e \"console.log('threads=' + process.env.NEXTEST_TEST_THREADS + ' tokens=' + process.env.BOUNDED_EXEC_CPU_TOKENS)\" # nextest run";
837
980
 
838
981
  const result = await runBoundedExec(
839
- { command, timeoutMs: 5_000, parallelism: 8, env: { NEXTEST_TEST_THREADS: '64' } },
982
+ { command, timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS, parallelism: 8, env: { NEXTEST_TEST_THREADS: '64' } },
840
983
  workspace.context,
841
984
  createProcessTreeKiller(),
842
985
  null,
@@ -867,7 +1010,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
867
1010
  const result = await runBoundedExec(
868
1011
  {
869
1012
  command: 'node -e "console.log(\'threads=\' + process.env.NEXTEST_TEST_THREADS)" # nextest run',
870
- timeoutMs: 5_000,
1013
+ timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
871
1014
  env: { NEXTEST_TEST_THREADS: '64' },
872
1015
  },
873
1016
  workspace.context,
@@ -892,7 +1035,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
892
1035
  };
893
1036
 
894
1037
  const result = await runBoundedExec(
895
- { command: 'node -e "console.log(\'ran\')"', timeoutMs: 5_000 },
1038
+ { command: 'node -e "console.log(\'ran\')"', timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS },
896
1039
  workspace.context,
897
1040
  createProcessTreeKiller(),
898
1041
  null,
@@ -903,6 +1046,52 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
903
1046
  expect(result.terminalOutput).toBe('ran\n');
904
1047
  });
905
1048
 
1049
+ it('resolves the cowshed a run uses, in full precedence order', () => {
1050
+ const root = '/workspace';
1051
+ // The executor never chdirs, so an empty PATH component names its actual spawn cwd,
1052
+ // deliberately not the workspace root: the two directories differ on purpose here.
1053
+ const cwd = '/elsewhere';
1054
+ // Every level present at once, removed one at a time from the top: each step must fall
1055
+ // to exactly the next level, so the order itself is the assertion, not each level alone.
1056
+ const everywhere = new Set([
1057
+ '/opt/bin/cowshed',
1058
+ '/elsewhere/cowshed',
1059
+ '/workspace/packages/cowshed/bin/cowshed',
1060
+ '/workspace/node_modules/@smoothbricks/cowshed/bin/cowshed',
1061
+ ]);
1062
+ const present = (path: string) => everywhere.has(path);
1063
+ const select = (named?: string, inherited?: string, pathValue?: string) =>
1064
+ resolveCowshedExecutable(root, named, inherited, pathValue, cwd, present);
1065
+
1066
+ // 1. A target-named executable wins, made absolute against the workspace root.
1067
+ expect(select('bin/cowshed', '/valid/bin/cowshed', '/opt/bin')).toBe('/workspace/bin/cowshed');
1068
+ // 2. An explicit inherited executable outranks PATH and the workspace launcher — the
1069
+ // precedence createHostCommands always had — and keeps the coordinate the parent
1070
+ // gave it: absolute verbatim, relative untouched for spawn's own resolution,
1071
+ // never retargeted to the workspace root.
1072
+ expect(select(undefined, '/valid/bin/cowshed', '/opt/bin')).toBe('/valid/bin/cowshed');
1073
+ expect(select(undefined, 'bin/cowshed', '/opt/bin')).toBe('bin/cowshed');
1074
+ // 3. An installed cowshed on PATH wins (undefined: the bare name spawn already finds it).
1075
+ expect(select(undefined, undefined, '/opt/bin')).toBeUndefined();
1076
+ // 3b. An empty PATH component (the leading ':' below) names the current directory, as
1077
+ // POSIX spawn resolution does — here the executor's cwd, not the workspace root:
1078
+ // a cowshed in the cwd is installed too, and one merely in the root is not.
1079
+ expect(select(undefined, undefined, ':/usr/bin')).toBeUndefined();
1080
+ everywhere.delete('/elsewhere/cowshed');
1081
+ everywhere.add('/workspace/cowshed');
1082
+ expect(select(undefined, undefined, ':/usr/bin')).toBe('/workspace/packages/cowshed/bin/cowshed');
1083
+ everywhere.delete('/workspace/cowshed');
1084
+ // 4. Only when PATH provides none does the workspace's own launcher answer.
1085
+ expect(select(undefined, undefined, '/usr/bin')).toBe('/workspace/packages/cowshed/bin/cowshed');
1086
+ // 5. A linked workspace's launcher answers when the workspace has none of its own.
1087
+ everywhere.delete('/workspace/packages/cowshed/bin/cowshed');
1088
+ expect(select(undefined, undefined, '/usr/bin')).toBe('/workspace/node_modules/@smoothbricks/cowshed/bin/cowshed');
1089
+ // 6. No cowshed anywhere keeps the bare name, so the spawn failure names the missing tool.
1090
+ everywhere.clear();
1091
+ expect(select(undefined, undefined, '/usr/bin')).toBeUndefined();
1092
+ expect(select(undefined, undefined, undefined)).toBeUndefined();
1093
+ });
1094
+
906
1095
  it('sizes a runner to the grant its parent pool lent, takes nothing from the host, and returns nothing to it', async () => {
907
1096
  const workspace = await createWorkspace();
908
1097
  const asked: number[] = [];
@@ -919,7 +1108,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
919
1108
  runBoundedExec(
920
1109
  {
921
1110
  command: 'node -e "console.log(\'tokens=\' + process.env.BOUNDED_EXEC_CPU_TOKENS)" # nextest run',
922
- timeoutMs: 5_000,
1111
+ timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
923
1112
  parallelism: 8,
924
1113
  },
925
1114
  workspace.context,
@@ -945,7 +1134,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
945
1134
  runBoundedExec(
946
1135
  {
947
1136
  command: 'node -e "console.log(\'threads=\' + process.env.NEXTEST_TEST_THREADS)" # nextest run',
948
- timeoutMs: 5_000,
1137
+ timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
949
1138
  parallelism: 2,
950
1139
  },
951
1140
  workspace.context,
@@ -973,7 +1162,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
973
1162
  };
974
1163
 
975
1164
  const result = await runBoundedExec(
976
- { command: 'node -e "console.log(1)" # nextest run', timeoutMs: 5_000, parallelism: 8 },
1165
+ { command: 'node -e "console.log(1)" # nextest run', timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS, parallelism: 8 },
977
1166
  workspace.context,
978
1167
  createProcessTreeKiller(),
979
1168
  null,
@@ -1008,7 +1197,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
1008
1197
 
1009
1198
  const borrowed = await withPool(3, (pool) =>
1010
1199
  runBoundedExec(
1011
- { command, timeoutMs: 5_000, parallelism: 8 },
1200
+ { command, timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS, parallelism: 8 },
1012
1201
  { ...workspace.context, record: { task, directory: borrowedDir } },
1013
1202
  createProcessTreeKiller(),
1014
1203
  null,
@@ -1017,7 +1206,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
1017
1206
  ),
1018
1207
  );
1019
1208
  const owned = await runBoundedExec(
1020
- { command, timeoutMs: 5_000, parallelism: 8 },
1209
+ { command, timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS, parallelism: 8 },
1021
1210
  { ...workspace.context, record: { task, directory: ownedDir } },
1022
1211
  createProcessTreeKiller(),
1023
1212
  null,
@@ -1039,41 +1228,45 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
1039
1228
  expect(ownedRecord).not.toHaveProperty('cpu.inherited');
1040
1229
  });
1041
1230
 
1042
- it('records borrowed grant loss during timeout termination as an ordinary failure, not a bound', async () => {
1231
+ it('keeps a command running when its borrowed grant closes, and judges it by its own exit', async () => {
1043
1232
  if (process.platform === 'win32') return;
1044
1233
  const workspace = await createWorkspace();
1045
- const directory = join(workspace.root, 'lost-during-timeout');
1046
- const ignoring = ignoresTerm(workspace.root);
1047
- const calls: NodeJS.Signals[] = [];
1048
- const result = await withPool(1, (pool) =>
1049
- runBoundedExec(
1050
- {
1051
- command: ignoring.command,
1052
- timeoutMs: 50,
1053
- killAfterMs: 50,
1054
- },
1055
- { ...workspace.context, record: { task: { id: 'lost-during-timeout', hash: null }, directory } },
1234
+ const directory = join(workspace.root, 'outlives-its-grant');
1235
+ const started = join(workspace.root, 'started.fifo');
1236
+ const go = join(workspace.root, 'go.fifo');
1237
+ expect(spawnSync('mkfifo', [started, go]).status).toBe(0);
1238
+ // The run says the grant ended on its stderr; that line is the signal the close was heard while the command runs.
1239
+ const { promise: heard, resolve: hear } = Promise.withResolvers<void>();
1240
+ process.stderr.write = captureWrite([], (text) => {
1241
+ if (text.includes('cpu-tokens grant ended')) hear();
1242
+ });
1243
+ const result = await withPool(1, async (pool) => {
1244
+ const running = runBoundedExec(
1056
1245
  {
1057
- async kill(pid, signal) {
1058
- if (signal === 'SIGTERM') {
1059
- await ignoring.armed();
1060
- const closed = await pool.close();
1061
- if (!closed.ok) throw new Error(closed.error);
1062
- }
1063
- calls.push(signal);
1064
- await createProcessTreeKiller().kill(pid, signal);
1065
- },
1246
+ command: `sh -c 'echo started > "$0"; read -r line < "$1"' '${started}' '${go}'`,
1247
+ timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
1066
1248
  },
1249
+ { ...workspace.context, record: { task: { id: 'outlives-its-grant', hash: null }, directory } },
1250
+ createProcessTreeKiller(),
1067
1251
  null,
1068
1252
  null,
1069
1253
  inheritedFrom(pool),
1070
- ),
1254
+ );
1255
+ // The command has started and waits on `go` while its borrowed grant closes under it.
1256
+ expect(await readToEnd(started)).toBe('started\n');
1257
+ const closed = await pool.close();
1258
+ if (!closed.ok) throw new Error(closed.error);
1259
+ await heard;
1260
+ await writeFile(go, 'go\n');
1261
+ return running;
1262
+ });
1263
+ expect(result.terminalOutput).toContain(
1264
+ 'cowshed: cpu-tokens grant ended: the parent pool closed its 1-token grant while the command ran; it runs on over budget until it ends',
1071
1265
  );
1072
- expect(result.success).toBe(false);
1073
- expect(calls).toEqual(['SIGTERM', 'SIGKILL']);
1074
- expect(result.terminalOutput).toContain('cpu-tokens grant lost');
1266
+ expect(result.terminalOutput).not.toContain('Force-killing');
1267
+ expect(result.success).toBe(true);
1075
1268
  const record: unknown = JSON.parse(await readFile(join(directory, VERDICT_FILE), 'utf8'));
1076
- expect(record).toHaveProperty('verdict.outcome', 'failed');
1269
+ expect(record).toHaveProperty('verdict.outcome', 'passed');
1077
1270
  });
1078
1271
 
1079
1272
  it('asks the host budget when a parent pool refuses its descriptor, and says why', async () => {
@@ -1089,7 +1282,11 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
1089
1282
  const gone = { socket: join(workspace.root, 'gone.sock'), tokens: 3 };
1090
1283
 
1091
1284
  const result = await runBoundedExec(
1092
- { command: 'node -e "console.log(\'ran\')" # nextest run', timeoutMs: 5_000, parallelism: 8 },
1285
+ {
1286
+ command: 'node -e "console.log(\'ran\')" # nextest run',
1287
+ timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
1288
+ parallelism: 8,
1289
+ },
1093
1290
  workspace.context,
1094
1291
  createProcessTreeKiller(),
1095
1292
  null,
@@ -1111,7 +1308,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
1111
1308
  const result = await runBoundedExec(
1112
1309
  {
1113
1310
  command: 'node -e "console.log(\'tokens=\' + process.env.BOUNDED_EXEC_CPU_TOKENS)"',
1114
- timeoutMs: 5_000,
1311
+ timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
1115
1312
  env: { BOUNDED_EXEC_CPU_TOKENS: '977' },
1116
1313
  },
1117
1314
  workspace.context,
@@ -1139,7 +1336,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
1139
1336
  const result = await runBoundedExec(
1140
1337
  {
1141
1338
  command: 'node -e "console.log(\'tokens=\' + process.env.BOUNDED_EXEC_CPU_TOKENS)" # nextest run',
1142
- timeoutMs: 5_000,
1339
+ timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
1143
1340
  parallelism: 8,
1144
1341
  env: { BOUNDED_EXEC_CPU_TOKENS: '977' },
1145
1342
  },
@@ -1176,7 +1373,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
1176
1373
  };
1177
1374
  try {
1178
1375
  const result = await runBoundedExec(
1179
- { command: 'touch ran.marker', timeoutMs: 5_000 },
1376
+ { command: 'touch ran.marker', timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS },
1180
1377
  {
1181
1378
  ...workspace.context,
1182
1379
  record: { task: { id: 'refused', hash: null }, directory: join(workspace.root, 'refused') },
@@ -1214,7 +1411,11 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
1214
1411
  };
1215
1412
  const unbudgeted = await withPool(3, (pool) =>
1216
1413
  runBoundedExec(
1217
- { command: 'node probe.js', timeoutMs: 10_000, env: { ...pool.environment, [CPU_TOKENS_ENV]: '9' } },
1414
+ {
1415
+ command: 'node probe.js',
1416
+ timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
1417
+ env: { ...pool.environment, [CPU_TOKENS_ENV]: '9' },
1418
+ },
1218
1419
  workspace.context,
1219
1420
  createProcessTreeKiller(),
1220
1421
  null,
@@ -1228,7 +1429,11 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
1228
1429
  const granted = await withPool(3, (pool) => {
1229
1430
  injected = inheritedFrom(pool).socket;
1230
1431
  return runBoundedExec(
1231
- { command: 'node probe.js', timeoutMs: 10_000, env: { ...pool.environment, [CPU_TOKENS_ENV]: '9' } },
1432
+ {
1433
+ command: 'node probe.js',
1434
+ timeoutMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
1435
+ env: { ...pool.environment, [CPU_TOKENS_ENV]: '9' },
1436
+ },
1232
1437
  workspace.context,
1233
1438
  createProcessTreeKiller(),
1234
1439
  null,
@@ -1260,6 +1465,25 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
1260
1465
  });
1261
1466
  });
1262
1467
 
1468
+ describe('createNoSpaceWatch', () => {
1469
+ it("finds nextest's ENOSPC split across chunks, past colour codes, and reports it once", () => {
1470
+ // nextest's own report of a failed archive extraction into a full RAM lease, from a downstream land gate.
1471
+ const watch = createNoSpaceWatch();
1472
+ expect(watch.scan('stderr', 'Caused by:\n No space left')).toBeNull();
1473
+ expect(watch.scan('stdout', ' on device (os error 28)\n')).toBeNull();
1474
+ expect(watch.scan('stderr', ' on device \u001b[1m(os error 28)\u001b[0m\nCommand exited')).toBe(
1475
+ 'No space left on device (os error 28)',
1476
+ );
1477
+ expect(watch.scan('stderr', 'ENOSPC: no space left on device, write\n')).toBeNull();
1478
+ });
1479
+
1480
+ it("finds Bun's errno name, and not a word that merely contains it", () => {
1481
+ const watch = createNoSpaceWatch();
1482
+ expect(watch.scan('stdout', 'XENOSPCY is not an errno\n')).toBeNull();
1483
+ expect(watch.scan('stdout', ' code: "ENOSPC"\n')).toBe('code: "ENOSPC"');
1484
+ });
1485
+ });
1486
+
1263
1487
  interface WorkspaceFixture {
1264
1488
  root: string;
1265
1489
  context: BoundedExecContext;
@@ -1320,17 +1544,18 @@ async function armedPid(path: string, pgid: number, subject: string): Promise<nu
1320
1544
  }
1321
1545
 
1322
1546
  /**
1323
- * A command that ignores SIGTERM, a disposition its `sleep` inherits, and `armed`, which resolves once it does. A
1324
- * killer that awaits `armed` before the graceful signal sends it to a command that cannot die of it: a bound that fires
1325
- * before the command's shell has even run `trap` (a 250 ms timer lost that race in a loaded gate) ends it on the
1326
- * default disposition, and the escalation under test never runs. The command's FIFO write waits for the reader, so
1327
- * the announcement can neither precede the trap nor be missed.
1547
+ * A command that ignores SIGTERM, a disposition its `tail` inherits, and never exits on its own, so only the escalation
1548
+ * can end it; and `armed`, which resolves once it ignores SIGTERM. A killer that awaits `armed` before the graceful
1549
+ * signal sends it to a command that cannot die of it: a bound that fires before the command's shell has even run
1550
+ * `trap` (a 250 ms timer lost that race in a loaded gate) ends it on the default disposition, and the escalation under
1551
+ * test never runs. The command's FIFO write waits for the reader, so the announcement can neither precede the trap nor
1552
+ * be missed.
1328
1553
  */
1329
1554
  function ignoresTerm(root: string): { command: string; armed: () => Promise<void> } {
1330
1555
  const fifo = join(root, 'armed.fifo');
1331
1556
  expect(spawnSync('mkfifo', [fifo]).status).toBe(0);
1332
1557
  return {
1333
- command: `sh -c 'trap "" TERM; echo armed > "$0"; exec sleep 5' '${fifo}'`,
1558
+ command: `sh -c 'trap "" TERM; echo armed > "$0"; exec tail -f /dev/null' '${fifo}'`,
1334
1559
  armed: async () => {
1335
1560
  expect(await readToEnd(fifo)).toBe('armed\n');
1336
1561
  },
@@ -1355,11 +1580,12 @@ function leaseAt(directory: string): RamTempLease {
1355
1580
  return { directory, mountpoint: dirname(directory), capacityBytes: 1024 * 1024 * 1024 };
1356
1581
  }
1357
1582
 
1358
- /** A temp volume whose answers are fixed, recording which leases were reaped and released, in order. */
1583
+ /** A temp volume whose answers are fixed, recording which leases were reaped and released, and when space was read. */
1359
1584
  function scriptedVolume(
1360
1585
  acquisition: Result<RamTempAcquisition, RamTempError>,
1361
1586
  full: RamTempError | null = null,
1362
1587
  leftovers: string[] = [],
1588
+ space: TempVolumeSpace | null = null,
1363
1589
  ): TempVolume & { released: string[]; order: string[] } {
1364
1590
  const released: string[] = [];
1365
1591
  const order: string[] = [];
@@ -1377,16 +1603,22 @@ function scriptedVolume(
1377
1603
  return Promise.resolve({ ok: true, value: undefined });
1378
1604
  },
1379
1605
  fullness: () => Promise.resolve(full),
1606
+ space() {
1607
+ order.push('space');
1608
+ return Promise.resolve(space);
1609
+ },
1380
1610
  };
1381
1611
  }
1382
1612
 
1383
- function captureWrite(chunks: string[]): typeof process.stdout.write {
1613
+ function captureWrite(chunks: string[], onChunk?: (text: string) => void): typeof process.stdout.write {
1384
1614
  return ((
1385
1615
  chunk: string | Uint8Array,
1386
1616
  encodingOrCallback?: BufferEncoding | ((error?: Error | null) => void),
1387
1617
  callback?: (error?: Error | null) => void,
1388
1618
  ) => {
1389
- chunks.push(chunk.toString());
1619
+ const text = chunk.toString();
1620
+ chunks.push(text);
1621
+ onChunk?.(text);
1390
1622
  const cb = typeof encodingOrCallback === 'function' ? encodingOrCallback : callback;
1391
1623
  cb?.();
1392
1624
  return true;
@@ -1419,9 +1651,10 @@ function heldGrant(tokens: number, released: number[]): TokenGrant {
1419
1651
  };
1420
1652
  }
1421
1653
 
1422
- /** A parent's CPU pool of `tokens`, open while `body` runs. Its upstream never closes on its own. */
1654
+ /** A parent's CPU pool of `tokens`, open while `body` runs. Its upstream never closes on its own, and no run owns the
1655
+ * host waits its children report. */
1423
1656
  async function withPool<T>(tokens: number, body: (pool: CpuPool) => Promise<T>): Promise<T> {
1424
- const created = await createCpuPool(tokens, new Promise<void>(() => {}));
1657
+ const created = await createCpuPool(tokens, new Promise<void>(() => {}), { begin() {}, end() {} });
1425
1658
  if (!created.ok) {
1426
1659
  throw new Error(`cpu pool: ${created.error}`);
1427
1660
  }
@@ -1449,7 +1682,20 @@ function inheritedFrom(pool: CpuPool): InheritedCpuPool {
1449
1682
  return parent;
1450
1683
  }
1451
1684
 
1452
- /** The count a pool of `tokens` lends a run that wants `want`: min(want, tokens, ceil(cores / 2)). */
1685
+ /** The count a pool of `tokens` lends a run that wants `want`: min(want, tokens). */
1453
1686
  function lentCount(want: number, tokens: number): number {
1454
- return Math.min(want, tokens, Math.ceil(availableParallelism() / 2));
1687
+ return Math.min(want, tokens);
1688
+ }
1689
+
1690
+ /** Every process left in group `pgid` is a zombie: `ps` lists no member in any other state. */
1691
+ function onlyZombiesIn(pgid: number): boolean {
1692
+ const listed = spawnSync('ps', ['-A', '-o', 'pgid=,stat='], { encoding: 'utf8' });
1693
+ if (listed.status !== 0) {
1694
+ return false;
1695
+ }
1696
+ return listed.stdout
1697
+ .split('\n')
1698
+ .map((line) => line.trim().split(/\s+/))
1699
+ .filter(([group]) => Number(group) === pgid)
1700
+ .every(([, state]) => state?.startsWith('Z') === true);
1455
1701
  }