@smoothbricks/nx-plugin 0.4.27 → 0.4.29

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (124) hide show
  1. package/README.md +203 -83
  2. package/dist/bounded-test-policy.d.ts +10 -3
  3. package/dist/bounded-test-policy.d.ts.map +1 -1
  4. package/dist/bounded-test-policy.js +19 -25
  5. package/dist/cargo-closure-input.d.ts +12 -0
  6. package/dist/cargo-closure-input.d.ts.map +1 -1
  7. package/dist/cargo-closure-input.js +29 -7
  8. package/dist/cargo-source-hash.d.ts.map +1 -1
  9. package/dist/cargo-source-hash.js +42 -19
  10. package/dist/ensure-built-receipt.d.ts +34 -6
  11. package/dist/ensure-built-receipt.d.ts.map +1 -1
  12. package/dist/ensure-built-receipt.js +265 -159
  13. package/dist/ensure-built-witness.d.ts +27 -2
  14. package/dist/ensure-built-witness.d.ts.map +1 -1
  15. package/dist/ensure-built-witness.js +2176 -15
  16. package/dist/ensure-built.d.ts +99 -1
  17. package/dist/ensure-built.d.ts.map +1 -1
  18. package/dist/ensure-built.js +34 -53
  19. package/dist/executors/bounded-exec/cpu-pool.d.ts +16 -5
  20. package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -1
  21. package/dist/executors/bounded-exec/cpu-pool.js +43 -19
  22. package/dist/executors/bounded-exec/executor.d.ts +11 -2
  23. package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
  24. package/dist/executors/bounded-exec/executor.js +139 -39
  25. package/dist/executors/bounded-exec/ram-temp.d.ts +17 -11
  26. package/dist/executors/bounded-exec/ram-temp.d.ts.map +1 -1
  27. package/dist/executors/bounded-exec/ram-temp.js +27 -13
  28. package/dist/generators/bun-test-tracing/generator.js +2 -2
  29. package/dist/graph-input-contract.d.ts +98 -0
  30. package/dist/graph-input-contract.d.ts.map +1 -0
  31. package/dist/graph-input-contract.js +624 -0
  32. package/dist/index.d.ts +12 -5
  33. package/dist/index.d.ts.map +1 -1
  34. package/dist/index.js +17 -3
  35. package/dist/managed-files/ci-workflow.d.ts.map +1 -1
  36. package/dist/managed-files/ci-workflow.js +5 -2
  37. package/dist/managed-files/publish-workflow.d.ts +11 -1
  38. package/dist/managed-files/publish-workflow.d.ts.map +1 -1
  39. package/dist/producer-receipts.d.ts +19 -0
  40. package/dist/producer-receipts.d.ts.map +1 -0
  41. package/dist/producer-receipts.js +921 -0
  42. package/dist/testing.d.ts +8 -0
  43. package/dist/testing.d.ts.map +1 -1
  44. package/dist/testing.js +58 -0
  45. package/managed/raw/patches/nx@23.2.1.patch +328 -50
  46. package/managed/raw/tooling/direnv/developer-links.ts +147 -21
  47. package/managed/raw/tooling/direnv/devenv.smoo.nix +7 -7
  48. package/managed/raw/tooling/direnv/github-actions-bootstrap.sh +14 -5
  49. package/managed/raw/tooling/direnv/linked-snapshots.ts +206 -40
  50. package/managed/raw/tooling/direnv/repo-path +15 -2
  51. package/managed/raw/tooling/direnv/setup-environment.ts +363 -125
  52. package/managed/templates/github/actions/cache-node-modules/action.yml +9 -0
  53. package/managed/templates/github/actions/setup-devenv/action.yml +63 -7
  54. package/nextest.toml +1 -1
  55. package/package.json +11 -6
  56. package/src/__tests__/fixture-nx-env.ts +7 -0
  57. package/src/__tests__/owned-installation.ts +174 -0
  58. package/src/bin/smoo-nx-bound-failures.test.ts +1 -1
  59. package/src/bounded-test-policy.test.ts +36 -22
  60. package/src/bounded-test-policy.ts +19 -27
  61. package/src/bun-bootstrap.test.ts +1 -1
  62. package/src/cargo-closure-input.test.ts +60 -11
  63. package/src/cargo-closure-input.ts +28 -6
  64. package/src/cargo-resolve-cache.test.ts +140 -25
  65. package/src/cargo-source-hash.test.ts +78 -3
  66. package/src/cargo-source-hash.ts +48 -30
  67. package/src/cross-check-policy.test.ts +2 -2
  68. package/src/ensure-built-receipt.test.ts +111 -16
  69. package/src/ensure-built-receipt.ts +251 -149
  70. package/src/ensure-built-witness.test.ts +197 -13
  71. package/src/ensure-built-witness.ts +187 -15
  72. package/src/ensure-built.test.ts +156 -102
  73. package/src/ensure-built.ts +114 -64
  74. package/src/executors/bounded-exec/cpu-pool.test.ts +116 -32
  75. package/src/executors/bounded-exec/cpu-pool.ts +55 -17
  76. package/src/executors/bounded-exec/cpu-tokens.test.ts +2 -2
  77. package/src/executors/bounded-exec/executor-host.test.ts +51 -41
  78. package/src/executors/bounded-exec/executor.test.ts +155 -35
  79. package/src/executors/bounded-exec/executor.ts +156 -45
  80. package/src/executors/bounded-exec/progress.test.ts +1 -1
  81. package/src/executors/bounded-exec/ram-temp-host.test.ts +46 -17
  82. package/src/executors/bounded-exec/ram-temp.test.ts +65 -3
  83. package/src/executors/bounded-exec/ram-temp.ts +32 -16
  84. package/src/executors/bounded-exec/verdict.test.ts +17 -12
  85. package/src/generators/bounded-test-targets/generator.test.ts +1 -3
  86. package/src/generators/bun-test-tracing/generator.test.ts +2 -2
  87. package/src/generators/bun-test-tracing/generator.ts +2 -2
  88. package/src/graph-input-contract.test.ts +371 -0
  89. package/src/graph-input-contract.ts +605 -0
  90. package/src/index.test.ts +3 -3
  91. package/src/index.ts +24 -3
  92. package/src/linked-snapshots.nx.test.ts +112 -30
  93. package/src/managed-files/ci-workflow.ts +5 -2
  94. package/src/managed-files/publish-workflow.ts +12 -1
  95. package/src/managed-files/repo-path.test.ts +44 -1
  96. package/src/managed-files/setup-devenv-free-disk.test.ts +95 -0
  97. package/src/nextest-code-bound.test.ts +23 -17
  98. package/src/nextest-partitions.test.ts +7 -1
  99. package/src/nx-cache-restore-mtime.test.ts +2 -2
  100. package/src/nx-daemon-claim.test.ts +5 -5
  101. package/src/nx-daemon-idle-cpu.test.ts +0 -1
  102. package/src/nx-daemon-idle-policy.test.ts +56 -0
  103. package/src/nx-daemon-ignore-files.test.ts +2 -2
  104. package/src/nx-daemon-validity.test.ts +31 -9
  105. package/src/nx-dot-input-policy.test.ts +5 -5
  106. package/src/nx-files-archive-racy.test.ts +1 -1
  107. package/src/nx-incomplete-run-summary.test.ts +1 -1
  108. package/src/nx-plugin-load-errors.test.ts +2 -2
  109. package/src/nx-project-graph-errors.test.ts +70 -0
  110. package/src/nx-skipped-task-exit.test.ts +2 -2
  111. package/src/nx-store-resolution.test.ts +7 -25
  112. package/src/nx-task-output-lease.test.ts +3 -3
  113. package/src/nx-use-local.test.ts +7 -5
  114. package/src/producer-receipts.test.ts +164 -0
  115. package/src/producer-receipts.ts +631 -0
  116. package/src/runtime-input.test.ts +1 -1
  117. package/src/serial-group-wiring.test.ts +4 -4
  118. package/src/task-results.test.ts +18 -16
  119. package/src/test-timings.test.ts +2 -2
  120. package/src/testing.test.ts +5 -5
  121. package/src/testing.ts +59 -0
  122. package/src/typecheck-test-policy.test.ts +1 -1
  123. package/src/typescript-resolution.test.ts +76 -0
  124. package/test-timings.json +5 -0
@@ -2,7 +2,7 @@ import { afterEach, describe, expect, it } from 'bun:test';
2
2
  import { spawnSync } from 'node:child_process';
3
3
  import { access, mkdir, mkdtemp, open, readFile, rm, writeFile } from 'node:fs/promises';
4
4
  import { createServer } from 'node:net';
5
- import { availableParallelism, tmpdir } from 'node:os';
5
+ import { tmpdir } from 'node:os';
6
6
  import { dirname, join } from 'node:path';
7
7
  import { onProgressClock } from '../../__tests__/progress-clock.js';
8
8
  import { ownedSocketDirectory } from '../../testing.js';
@@ -12,6 +12,7 @@ import {
12
12
  type BoundedExecContext,
13
13
  createProcessTreeKiller,
14
14
  type ProcessTreeKiller,
15
+ resolveCowshedExecutable,
15
16
  runBoundedExec,
16
17
  type TempVolume,
17
18
  } from './executor.js';
@@ -44,7 +45,10 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
44
45
  try {
45
46
  process.kill(-pid, 'SIGKILL');
46
47
  } catch (cause) {
47
- if (!(cause instanceof Error && 'code' in cause && cause.code === 'ESRCH')) {
48
+ const code = cause instanceof Error && 'code' in cause ? cause.code : undefined;
49
+ // Darwin answers EPERM, not ESRCH, for a group whose only members are unreaped zombies (a killed
50
+ // descendant waiting for launchd to reap it): such a group has nothing left to kill.
51
+ if (code !== 'ESRCH' && !(code === 'EPERM' && onlyZombiesIn(pid))) {
48
52
  throw new Error(`cleanup owned readiness process group pgid=${pid} failed`, { cause });
49
53
  }
50
54
  } finally {
@@ -186,6 +190,56 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
186
190
  expect(result.terminalOutput).not.toContain('fake tests::quick (cwd=');
187
191
  });
188
192
 
193
+ it("stops the idle clock while a descendant's work is queued for host CPU tokens, and runs it on after", async () => {
194
+ const workspace = await createWorkspace();
195
+ // The runner finishes a test, then says through its pool that work it started is queued for host tokens, ends that
196
+ // wait and goes silent. While the wait is held the clock moves well past the bound and nothing fires; once the wait
197
+ // ends the same move ends the run.
198
+ await workspace.write(
199
+ 'queued-runner.js',
200
+ [
201
+ "const { connect } = require('node:net');",
202
+ "console.error(' PASS [ 0.010s] (1/2) fake tests::quick');",
203
+ `const socket = connect(process.env.${CPU_POOL_ENV});`,
204
+ `socket.on('connect', () => socket.write('{"op":"host-wait"}\\n'));`,
205
+ "socket.once('data', (reply) => { console.log('host wait ' + reply.toString().trim()); socket.end(); });",
206
+ "socket.on('close', () => setInterval(() => {}, 60_000));",
207
+ '',
208
+ ].join('\n'),
209
+ );
210
+ const budget: CpuBudget = {
211
+ async take() {
212
+ return heldGrant(1, []);
213
+ },
214
+ };
215
+
216
+ const { result } = await onProgressClock(
217
+ (chunk) => chunk.includes('bounded-exec: idle clock'),
218
+ 1_000,
219
+ () =>
220
+ runBoundedExec(
221
+ { command: 'node queued-runner.js', idleTimeoutMs: 400, killAfterMs: 0 },
222
+ workspace.context,
223
+ createProcessTreeKiller(),
224
+ null,
225
+ budget,
226
+ ),
227
+ );
228
+
229
+ const output = result.terminalOutput;
230
+ expect(result.success).toBe(false);
231
+ expect(output).toContain('host wait {"ok":true,"wait":"held"}');
232
+ const stopped = output.indexOf(
233
+ "bounded-exec: idle clock stopped: a descendant's work is queued for host CPU tokens",
234
+ );
235
+ const restarted = output.indexOf('bounded-exec: idle clock restarted');
236
+ const expired = output.indexOf('Command made no progress: no test finished for 400ms (idleTimeoutMs=400)');
237
+ // The 1000 ms move while the wait was held fired nothing; the one after it ended the run.
238
+ expect(stopped).toBeGreaterThan(-1);
239
+ expect(restarted).toBeGreaterThan(stopped);
240
+ expect(expired).toBeGreaterThan(restarted);
241
+ });
242
+
189
243
  it('emits a completed pre-expiry snapshot or an explicit incomplete capture without waiting for it', async () => {
190
244
  const workspace = await createWorkspace();
191
245
  // Child output drives the fake executor clock; the heartbeat period is not a timing oracle.
@@ -652,7 +706,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
652
706
  expect(await straggler).toBe('');
653
707
  expect(result.success).toBe(false);
654
708
  expect(result.terminalOutput).toContain('Force-killing what the timed-out command left in its process group');
655
- }, 60_000);
709
+ });
656
710
 
657
711
  it('runs the command with its RAM lease as TMPDIR, names held dead leases, and ends the lease after it exits', async () => {
658
712
  const workspace = await createWorkspace();
@@ -903,6 +957,52 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
903
957
  expect(result.terminalOutput).toBe('ran\n');
904
958
  });
905
959
 
960
+ it('resolves the cowshed a run uses, in full precedence order', () => {
961
+ const root = '/workspace';
962
+ // The executor never chdirs, so an empty PATH component names its actual spawn cwd,
963
+ // deliberately not the workspace root: the two directories differ on purpose here.
964
+ const cwd = '/elsewhere';
965
+ // Every level present at once, removed one at a time from the top: each step must fall
966
+ // to exactly the next level, so the order itself is the assertion, not each level alone.
967
+ const everywhere = new Set([
968
+ '/opt/bin/cowshed',
969
+ '/elsewhere/cowshed',
970
+ '/workspace/packages/cowshed/bin/cowshed',
971
+ '/workspace/node_modules/@smoothbricks/cowshed/bin/cowshed',
972
+ ]);
973
+ const present = (path: string) => everywhere.has(path);
974
+ const select = (named?: string, inherited?: string, pathValue?: string) =>
975
+ resolveCowshedExecutable(root, named, inherited, pathValue, cwd, present);
976
+
977
+ // 1. A target-named executable wins, made absolute against the workspace root.
978
+ expect(select('bin/cowshed', '/valid/bin/cowshed', '/opt/bin')).toBe('/workspace/bin/cowshed');
979
+ // 2. An explicit inherited executable outranks PATH and the workspace launcher — the
980
+ // precedence createHostCommands always had — and keeps the coordinate the parent
981
+ // gave it: absolute verbatim, relative untouched for spawn's own resolution,
982
+ // never retargeted to the workspace root.
983
+ expect(select(undefined, '/valid/bin/cowshed', '/opt/bin')).toBe('/valid/bin/cowshed');
984
+ expect(select(undefined, 'bin/cowshed', '/opt/bin')).toBe('bin/cowshed');
985
+ // 3. An installed cowshed on PATH wins (undefined: the bare name spawn already finds it).
986
+ expect(select(undefined, undefined, '/opt/bin')).toBeUndefined();
987
+ // 3b. An empty PATH component (the leading ':' below) names the current directory, as
988
+ // POSIX spawn resolution does — here the executor's cwd, not the workspace root:
989
+ // a cowshed in the cwd is installed too, and one merely in the root is not.
990
+ expect(select(undefined, undefined, ':/usr/bin')).toBeUndefined();
991
+ everywhere.delete('/elsewhere/cowshed');
992
+ everywhere.add('/workspace/cowshed');
993
+ expect(select(undefined, undefined, ':/usr/bin')).toBe('/workspace/packages/cowshed/bin/cowshed');
994
+ everywhere.delete('/workspace/cowshed');
995
+ // 4. Only when PATH provides none does the workspace's own launcher answer.
996
+ expect(select(undefined, undefined, '/usr/bin')).toBe('/workspace/packages/cowshed/bin/cowshed');
997
+ // 5. A linked workspace's launcher answers when the workspace has none of its own.
998
+ everywhere.delete('/workspace/packages/cowshed/bin/cowshed');
999
+ expect(select(undefined, undefined, '/usr/bin')).toBe('/workspace/node_modules/@smoothbricks/cowshed/bin/cowshed');
1000
+ // 6. No cowshed anywhere keeps the bare name, so the spawn failure names the missing tool.
1001
+ everywhere.clear();
1002
+ expect(select(undefined, undefined, '/usr/bin')).toBeUndefined();
1003
+ expect(select(undefined, undefined, undefined)).toBeUndefined();
1004
+ });
1005
+
906
1006
  it('sizes a runner to the grant its parent pool lent, takes nothing from the host, and returns nothing to it', async () => {
907
1007
  const workspace = await createWorkspace();
908
1008
  const asked: number[] = [];
@@ -1039,41 +1139,45 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
1039
1139
  expect(ownedRecord).not.toHaveProperty('cpu.inherited');
1040
1140
  });
1041
1141
 
1042
- it('records borrowed grant loss during timeout termination as an ordinary failure, not a bound', async () => {
1142
+ it('keeps a command running when its borrowed grant closes, and judges it by its own exit', async () => {
1043
1143
  if (process.platform === 'win32') return;
1044
1144
  const workspace = await createWorkspace();
1045
- const directory = join(workspace.root, 'lost-during-timeout');
1046
- const ignoring = ignoresTerm(workspace.root);
1047
- const calls: NodeJS.Signals[] = [];
1048
- const result = await withPool(1, (pool) =>
1049
- runBoundedExec(
1050
- {
1051
- command: ignoring.command,
1052
- timeoutMs: 50,
1053
- killAfterMs: 50,
1054
- },
1055
- { ...workspace.context, record: { task: { id: 'lost-during-timeout', hash: null }, directory } },
1145
+ const directory = join(workspace.root, 'outlives-its-grant');
1146
+ const started = join(workspace.root, 'started.fifo');
1147
+ const go = join(workspace.root, 'go.fifo');
1148
+ expect(spawnSync('mkfifo', [started, go]).status).toBe(0);
1149
+ // The run says the grant ended on its stderr; that line is the signal the close was heard while the command runs.
1150
+ const { promise: heard, resolve: hear } = Promise.withResolvers<void>();
1151
+ process.stderr.write = captureWrite([], (text) => {
1152
+ if (text.includes('cpu-tokens grant ended')) hear();
1153
+ });
1154
+ const result = await withPool(1, async (pool) => {
1155
+ const running = runBoundedExec(
1056
1156
  {
1057
- async kill(pid, signal) {
1058
- if (signal === 'SIGTERM') {
1059
- await ignoring.armed();
1060
- const closed = await pool.close();
1061
- if (!closed.ok) throw new Error(closed.error);
1062
- }
1063
- calls.push(signal);
1064
- await createProcessTreeKiller().kill(pid, signal);
1065
- },
1157
+ command: `sh -c 'echo started > "$0"; read -r line < "$1"' '${started}' '${go}'`,
1158
+ timeoutMs: 20_000,
1066
1159
  },
1160
+ { ...workspace.context, record: { task: { id: 'outlives-its-grant', hash: null }, directory } },
1161
+ createProcessTreeKiller(),
1067
1162
  null,
1068
1163
  null,
1069
1164
  inheritedFrom(pool),
1070
- ),
1165
+ );
1166
+ // The command has started and waits on `go` while its borrowed grant closes under it.
1167
+ expect(await readToEnd(started)).toBe('started\n');
1168
+ const closed = await pool.close();
1169
+ if (!closed.ok) throw new Error(closed.error);
1170
+ await heard;
1171
+ await writeFile(go, 'go\n');
1172
+ return running;
1173
+ });
1174
+ expect(result.terminalOutput).toContain(
1175
+ 'cowshed: cpu-tokens grant ended: the parent pool closed its 1-token grant while the command ran; it runs on over budget until it ends',
1071
1176
  );
1072
- expect(result.success).toBe(false);
1073
- expect(calls).toEqual(['SIGTERM', 'SIGKILL']);
1074
- expect(result.terminalOutput).toContain('cpu-tokens grant lost');
1177
+ expect(result.terminalOutput).not.toContain('Force-killing');
1178
+ expect(result.success).toBe(true);
1075
1179
  const record: unknown = JSON.parse(await readFile(join(directory, VERDICT_FILE), 'utf8'));
1076
- expect(record).toHaveProperty('verdict.outcome', 'failed');
1180
+ expect(record).toHaveProperty('verdict.outcome', 'passed');
1077
1181
  });
1078
1182
 
1079
1183
  it('asks the host budget when a parent pool refuses its descriptor, and says why', async () => {
@@ -1380,13 +1484,15 @@ function scriptedVolume(
1380
1484
  };
1381
1485
  }
1382
1486
 
1383
- function captureWrite(chunks: string[]): typeof process.stdout.write {
1487
+ function captureWrite(chunks: string[], onChunk?: (text: string) => void): typeof process.stdout.write {
1384
1488
  return ((
1385
1489
  chunk: string | Uint8Array,
1386
1490
  encodingOrCallback?: BufferEncoding | ((error?: Error | null) => void),
1387
1491
  callback?: (error?: Error | null) => void,
1388
1492
  ) => {
1389
- chunks.push(chunk.toString());
1493
+ const text = chunk.toString();
1494
+ chunks.push(text);
1495
+ onChunk?.(text);
1390
1496
  const cb = typeof encodingOrCallback === 'function' ? encodingOrCallback : callback;
1391
1497
  cb?.();
1392
1498
  return true;
@@ -1419,9 +1525,10 @@ function heldGrant(tokens: number, released: number[]): TokenGrant {
1419
1525
  };
1420
1526
  }
1421
1527
 
1422
- /** A parent's CPU pool of `tokens`, open while `body` runs. Its upstream never closes on its own. */
1528
+ /** A parent's CPU pool of `tokens`, open while `body` runs. Its upstream never closes on its own, and no run owns the
1529
+ * host waits its children report. */
1423
1530
  async function withPool<T>(tokens: number, body: (pool: CpuPool) => Promise<T>): Promise<T> {
1424
- const created = await createCpuPool(tokens, new Promise<void>(() => {}));
1531
+ const created = await createCpuPool(tokens, new Promise<void>(() => {}), { begin() {}, end() {} });
1425
1532
  if (!created.ok) {
1426
1533
  throw new Error(`cpu pool: ${created.error}`);
1427
1534
  }
@@ -1449,7 +1556,20 @@ function inheritedFrom(pool: CpuPool): InheritedCpuPool {
1449
1556
  return parent;
1450
1557
  }
1451
1558
 
1452
- /** The count a pool of `tokens` lends a run that wants `want`: min(want, tokens, ceil(cores / 2)). */
1559
+ /** The count a pool of `tokens` lends a run that wants `want`: min(want, tokens). */
1453
1560
  function lentCount(want: number, tokens: number): number {
1454
- return Math.min(want, tokens, Math.ceil(availableParallelism() / 2));
1561
+ return Math.min(want, tokens);
1562
+ }
1563
+
1564
+ /** Every process left in group `pgid` is a zombie: `ps` lists no member in any other state. */
1565
+ function onlyZombiesIn(pgid: number): boolean {
1566
+ const listed = spawnSync('ps', ['-A', '-o', 'pgid=,stat='], { encoding: 'utf8' });
1567
+ if (listed.status !== 0) {
1568
+ return false;
1569
+ }
1570
+ return listed.stdout
1571
+ .split('\n')
1572
+ .map((line) => line.trim().split(/\s+/))
1573
+ .filter(([group]) => Number(group) === pgid)
1574
+ .every(([, state]) => state?.startsWith('Z') === true);
1455
1575
  }
@@ -1,12 +1,14 @@
1
1
  import { type ChildProcess, spawn } from 'node:child_process';
2
+ import { accessSync, constants } from 'node:fs';
2
3
  import { availableParallelism, userInfo } from 'node:os';
3
- import { isAbsolute, join, resolve as resolvePath } from 'node:path';
4
+ import { delimiter, isAbsolute, join, resolve as resolvePath } from 'node:path';
4
5
  import { workspaceDataDirectoryForWorkspace } from 'nx/src/utils/cache-directory.js';
5
6
 
6
7
  import {
7
8
  CPU_POOL_ENV,
8
9
  type CpuPool,
9
10
  createCpuPool,
11
+ type HostWaits,
10
12
  type InheritedCpuPool,
11
13
  readInheritedCpuPool,
12
14
  takeParentCpuTokens,
@@ -88,8 +90,6 @@ interface RunState {
88
90
  settled: boolean;
89
91
  expiry: ExpiredBound | null;
90
92
  forceKillNeeded: boolean;
91
- /** Set when a borrowed grant closed while the command ran, and the command was stopped as a lost grant. */
92
- grantLost: boolean;
93
93
  /** The sub-pool's close error when it did not close cleanly: the run fails. */
94
94
  poolCloseError: string | null;
95
95
  }
@@ -103,6 +103,52 @@ export interface ProcessTreeKiller {
103
103
  /** Where the command's TMPDIR comes from; `null` leaves the inherited one. */
104
104
  export type TempVolume = Pick<RamTempVolume, 'acquire' | 'release' | 'fullness' | 'reap'>;
105
105
 
106
+ /** Workspace-provided cowshed launchers, in the order a workspace that provides one names them. */
107
+ const WORKSPACE_COWSHED_LAUNCHERS = [
108
+ 'packages/cowshed/bin/cowshed',
109
+ 'node_modules/@smoothbricks/cowshed/bin/cowshed',
110
+ ] as const;
111
+
112
+ /**
113
+ * The cowshed a run's disk commands use, in precedence order: the target's named one, an
114
+ * explicit one inherited from the parent process, the installed one on PATH, and — only when
115
+ * PATH provides none — the launcher this workspace itself provides (its own cowshed package,
116
+ * or a linked one's bin). An environment with no cowshed anywhere keeps the bare name, so
117
+ * its spawn failure names the missing installed tool.
118
+ */
119
+ export function resolveCowshedExecutable(
120
+ root: string,
121
+ named: string | undefined,
122
+ inherited: string | undefined,
123
+ pathValue: string | undefined,
124
+ cwd: string,
125
+ exists: (path: string) => boolean = (path) => {
126
+ try {
127
+ accessSync(path, constants.X_OK);
128
+ return true;
129
+ } catch {
130
+ return false;
131
+ }
132
+ },
133
+ ): string | undefined {
134
+ // A target-named executable is workspace-relative or absolute against the workspace root.
135
+ // An inherited explicit keeps the coordinate the parent gave it, exactly as the host
136
+ // commands always spawned it: an absolute one verbatim, a relative or bare one for
137
+ // spawn's own resolution — retargeting it to the root would silently change the tool.
138
+ if (named !== undefined) return resolvePath(root, named);
139
+ if (inherited !== undefined) return inherited;
140
+ // An empty PATH component names the spawning process's current directory, as POSIX
141
+ // spawn resolution does — the executor never chdirs, so that is the caller's cwd,
142
+ // not the workspace root.
143
+ const onPath = (pathValue ?? '').split(delimiter).some((dir) => exists(join(dir === '' ? cwd : dir, 'cowshed')));
144
+ if (onPath) return undefined;
145
+ for (const candidate of WORKSPACE_COWSHED_LAUNCHERS) {
146
+ const launcher = join(root, candidate);
147
+ if (exists(launcher)) return launcher;
148
+ }
149
+ return undefined;
150
+ }
151
+
106
152
  export default function boundedExecExecutor(
107
153
  options: BoundedExecOptions,
108
154
  context: NxTaskContext,
@@ -119,7 +165,13 @@ export function runDefaultBoundedExec(
119
165
  // for its command alike. Nx resolves `{workspaceRoot}/…` in an option to a workspace-relative path, which the
120
166
  // command, starting in another directory, could not find: it gets the absolute one.
121
167
  const named = options.env?.[COWSHED_EXECUTABLE_ENV];
122
- const cowshed = named === undefined ? undefined : resolvePath(context.root, named);
168
+ const cowshed = resolveCowshedExecutable(
169
+ context.root,
170
+ named,
171
+ process.env[COWSHED_EXECUTABLE_ENV],
172
+ process.env.PATH,
173
+ process.cwd(),
174
+ );
123
175
  const run: BoundedExecOptions =
124
176
  cowshed === undefined ? options : { ...options, env: { ...options.env, [COWSHED_EXECUTABLE_ENV]: cowshed } };
125
177
  // Only macOS has `hdiutil ram://`; elsewhere the command keeps the inherited TMPDIR.
@@ -167,8 +219,9 @@ export function recordContext(context: NxTaskContext): BoundedExecContext {
167
219
  *
168
220
  * `inherited` is the parent's CPU pool (`cpu-pool.ts`) that lends this run its tokens. The run asks that pool first
169
221
  * and the host budget only when the pool refuses. Whatever it is granted, the run serves its own sub-pool to its
170
- * descendants, closed before the grant returns. A borrowed grant that closes while the command runs stops the command
171
- * as a lost grant, which is a failure.
222
+ * descendants, closed before the grant returns. A grant whose connection closes while the command runs (a gateway
223
+ * restart, or the lending parent's exit) ends its accounting, never the command: the command runs on over budget to
224
+ * its own exit, and the run says so.
172
225
  */
173
226
  export async function runBoundedExec(
174
227
  options: BoundedExecOptions,
@@ -194,7 +247,6 @@ export async function runBoundedExec(
194
247
  settled: false,
195
248
  expiry: null,
196
249
  forceKillNeeded: false,
197
- grantLost: false,
198
250
  poolCloseError: null,
199
251
  };
200
252
  const appendStdout = (chunk: Buffer | string): void => {
@@ -267,7 +319,7 @@ export async function runBoundedExec(
267
319
  let grant: GrantedTokens | null = null;
268
320
  let parentRefusal: string | null = null;
269
321
  if (inherited !== null) {
270
- const fromParent = await takeParentCpuTokens(inherited, demand.want, cores);
322
+ const fromParent = await takeParentCpuTokens(inherited, demand.want);
271
323
  if (fromParent.granted) {
272
324
  grant = fromParent;
273
325
  } else if (fromParent.cause === 'absent') {
@@ -318,8 +370,8 @@ export async function runBoundedExec(
318
370
  return cpuReleased;
319
371
  };
320
372
  // Records the run's verdict when it has a task to record as. An `expiry` is set only when a bound ended the run.
321
- // An `operational` run failed on its CPU budget (a lost grant, or a sub-pool that did not close): an ordinary
322
- // failure, whatever bound also fired, so it is never judged as a bound.
373
+ // An `operational` run failed on its CPU budget (a sub-pool that did not close): an ordinary failure, whatever
374
+ // bound also fired, so it is never judged as a bound.
323
375
  const recordRun = async (
324
376
  exitCode: number,
325
377
  expiry: ExpiredBound | null,
@@ -353,10 +405,26 @@ export async function runBoundedExec(
353
405
  );
354
406
  }
355
407
 
408
+ // Descendant work queued for host CPU tokens, as its pool hears it (`host-wait`): while any is held the idle clock
409
+ // is stopped, because a queued job is not an idle one (specs/cowshed/05_gateway.md, "The bounds start at the
410
+ // grant"). The clock it stops is made with the command below; a wait held before then is counted and honoured there.
411
+ let heldWaits = 0;
412
+ let idleClock: { stop(): void; restart(): void } | null = null;
413
+ const hostWaits: HostWaits = {
414
+ begin: () => {
415
+ heldWaits += 1;
416
+ if (heldWaits === 1) idleClock?.stop();
417
+ },
418
+ end: () => {
419
+ heldWaits -= 1;
420
+ if (heldWaits === 0) idleClock?.restart();
421
+ },
422
+ };
423
+
356
424
  // A granted run serves a sub-pool of its own grant, so its descendants draw from that and never from above it.
357
425
  // A pool that cannot be set up refuses the run: no command runs unbudgeted.
358
426
  if (grant !== null) {
359
- const created = await createCpuPool(grant.tokens, grant.closed);
427
+ const created = await createCpuPool(grant.tokens, grant.closed, hostWaits);
360
428
  if (!created.ok) {
361
429
  return refuse(`cowshed: cpu pool unavailable: ${created.error}; refusing to run the command\n`);
362
430
  }
@@ -400,14 +468,17 @@ export async function runBoundedExec(
400
468
  clearTimeout(captureTimer);
401
469
  idleCapture?.controller.abort();
402
470
  };
403
- const armIdleTimer = (): void => {
404
- if (idleTimeoutMs === undefined || state.settled || state.expiry !== null) {
405
- return;
406
- }
471
+ // What is left of the current no-progress interval. A finished test sets it back to `idleTimeoutMs`; a held host
472
+ // wait keeps it unspent: the clock is stopped while descendant work is queued for CPU tokens, and runs on from where
473
+ // it stopped once the last wait ends. `queuedMs` is the queued time this interval has not counted, for the report.
474
+ let idleLeftMs = idleTimeoutMs ?? 0;
475
+ let queuedMs = 0;
476
+ let queuedSinceMs: number | null = null;
477
+ const scheduleIdle = (limitMs: number, leftMs: number): void => {
407
478
  clearTimeout(idleTimer);
408
479
  cancelCapture();
409
480
  const capture: NonNullable<typeof idleCapture> = {
410
- expiresAtMs: Date.now() + idleTimeoutMs,
481
+ expiresAtMs: Date.now() + leftMs,
411
482
  controller: new AbortController(),
412
483
  result: { kind: 'pending', startedAtMs: null },
413
484
  };
@@ -445,12 +516,51 @@ export async function runBoundedExec(
445
516
  },
446
517
  );
447
518
  },
448
- Math.max(0, idleTimeoutMs - HANG_CAPTURE_MS),
449
- );
450
- idleTimer = setTimeout(
451
- () => expire({ kind: 'idle', limitMs: idleTimeoutMs, tests: progress.inFlight() }),
452
- idleTimeoutMs,
519
+ Math.max(0, leftMs - HANG_CAPTURE_MS),
453
520
  );
521
+ idleTimer = setTimeout(() => expire({ kind: 'idle', limitMs, tests: progress.inFlight() }, queuedMs), leftMs);
522
+ };
523
+ const armIdleTimer = (): void => {
524
+ if (idleTimeoutMs === undefined || state.settled || state.expiry !== null) {
525
+ return;
526
+ }
527
+ idleLeftMs = idleTimeoutMs;
528
+ queuedMs = 0;
529
+ if (queuedSinceMs !== null) {
530
+ queuedSinceMs = Date.now();
531
+ return;
532
+ }
533
+ scheduleIdle(idleTimeoutMs, idleLeftMs);
534
+ };
535
+ idleClock = {
536
+ stop: () => {
537
+ if (idleTimeoutMs === undefined || state.settled || state.expiry !== null || queuedSinceMs !== null) {
538
+ return;
539
+ }
540
+ const now = Date.now();
541
+ queuedSinceMs = now;
542
+ if (idleCapture !== undefined) {
543
+ idleLeftMs = Math.max(0, idleCapture.expiresAtMs - now);
544
+ }
545
+ clearTimeout(idleTimer);
546
+ cancelCapture();
547
+ idleCapture = undefined;
548
+ appendStderr(
549
+ `bounded-exec: idle clock stopped: a descendant's work is queued for host CPU tokens; ${idleLeftMs}ms of idleTimeoutMs=${idleTimeoutMs} left\n`,
550
+ );
551
+ },
552
+ restart: () => {
553
+ if (idleTimeoutMs === undefined || state.settled || state.expiry !== null || queuedSinceMs === null) {
554
+ return;
555
+ }
556
+ const waitedMs = Date.now() - queuedSinceMs;
557
+ queuedMs += waitedMs;
558
+ queuedSinceMs = null;
559
+ appendStderr(
560
+ `bounded-exec: idle clock restarted: no descendant's work is queued for host CPU tokens after ${waitedMs}ms; ${idleLeftMs}ms of idleTimeoutMs=${idleTimeoutMs} left\n`,
561
+ );
562
+ scheduleIdle(idleTimeoutMs, idleLeftMs);
563
+ },
454
564
  };
455
565
 
456
566
  // Only the CHILD's own output is read for progress. The diagnostics below go
@@ -522,8 +632,9 @@ export async function runBoundedExec(
522
632
  // the reader unable to tell a wedged toolchain from a loaded machine — the
523
633
  // two have opposite fixes.
524
634
  let termination: Promise<void> = Promise.resolve();
525
- const expire = (bound: ExpiredBound): void => {
526
- if (state.settled || state.expiry !== null || state.grantLost) {
635
+ // `queuedMs` is the time the idle interval did not count because descendant work was queued for host CPU tokens.
636
+ const expire = (bound: ExpiredBound, queuedMs = 0): void => {
637
+ if (state.settled || state.expiry !== null) {
527
638
  return;
528
639
  }
529
640
  state.expiry = bound;
@@ -532,7 +643,9 @@ export async function runBoundedExec(
532
643
  const elapsedMs = Date.now() - startedAt;
533
644
  appendStderr(
534
645
  bound.kind === 'idle'
535
- ? `\nCommand made no progress: no test finished for ${bound.limitMs}ms (idleTimeoutMs=${bound.limitMs}) after ${elapsedMs}ms of runtime; ${
646
+ ? `\nCommand made no progress: no test finished for ${bound.limitMs}ms (idleTimeoutMs=${bound.limitMs})${
647
+ queuedMs > 0 ? `, not counting ${queuedMs}ms queued for host CPU tokens,` : ''
648
+ } after ${elapsedMs}ms of runtime; ${
536
649
  bound.tests.length > 0
537
650
  ? `still running: ${bound.tests.join(', ')}`
538
651
  : 'the runner named no test still running'
@@ -601,25 +714,23 @@ export async function runBoundedExec(
601
714
  const totalTimer =
602
715
  timeoutMs === undefined ? undefined : setTimeout(() => expire({ kind: 'total', limitMs: timeoutMs }), timeoutMs);
603
716
  armIdleTimer();
717
+ // A wait its pool heard before the command started stops the clock it was not yet there to stop.
718
+ if (heldWaits > 0) idleClock.stop();
604
719
 
605
- // A borrowed grant ends with the ancestor that lent it, and the command runs on tokens that ancestor takes back
606
- // then. Stopping the command is a lost grant: a failure, not a bound. Our own release after the command settled
607
- // is not a loss; `cpuReleased` marks it.
608
- const onGrantLost = (): void => {
609
- if (state.settled || state.grantLost || cpuReleased !== null) {
610
- return;
611
- }
612
- state.grantLost = true;
613
- cancelCapture();
614
- appendStderr(
615
- `cowshed: cpu-tokens grant lost: its parent pool closed while the command ran; stopping it: ${command}\n`,
616
- );
617
- // A bound may already be terminating the still-running command. The loss remains an ordinary
618
- // operational failure, but that existing termination owns the signals and grace.
619
- if (state.expiry === null) termination = terminate();
620
- };
621
- if (grant?.inherited) {
622
- void grant.closed.then(onGrantLost);
720
+ // A grant's connection is a ledger entry, not the command's owner (05_gateway.md "Host CPU budget": a gateway
721
+ // restart forgets every grant, and what runs under it runs on over budget until it finishes). Its close, while the
722
+ // command runs, is said and changes nothing else; only a bound or a signal stops the command. Our own release after
723
+ // the command settled is not a loss; `cpuReleased` marks it.
724
+ if (grant !== null) {
725
+ const held = grant;
726
+ void held.closed.then(() => {
727
+ if (state.settled || cpuReleased !== null) {
728
+ return;
729
+ }
730
+ appendStderr(
731
+ `cowshed: cpu-tokens grant ended: the ${held.inherited ? 'parent pool' : 'gateway'} closed its ${held.tokens}-token grant while the command ran; it runs on over budget until it ends: ${command}\n`,
732
+ );
733
+ });
623
734
  }
624
735
 
625
736
  const { code: exitCode, signal: exitSignal } = await exitPromise;
@@ -648,9 +759,9 @@ export async function runBoundedExec(
648
759
  }
649
760
  await releaseLease();
650
761
 
651
- // A run whose grant was lost, or whose sub-pool did not close cleanly, fails even when the command exited 0, and
652
- // whatever bound also fired: its verdict is an ordinary failure, never a bound.
653
- const operationalFailure = state.grantLost || state.poolCloseError !== null;
762
+ // A run whose sub-pool did not close cleanly fails even when the command exited 0, and whatever bound also fired:
763
+ // its verdict is an ordinary failure, never a bound.
764
+ const operationalFailure = state.poolCloseError !== null;
654
765
  await recordRun(code, state.expiry, Date.now() - startedAt, operationalFailure);
655
766
 
656
767
  return {
@@ -61,7 +61,7 @@ describe('run progress', () => {
61
61
 
62
62
  describe('progress report', () => {
63
63
  it('asks a lone bun test for dots, keeping the runtime the target named', () => {
64
- expect(withProgressReport('bun test --timeout=30000 src')).toBe('bun test --dots --timeout=30000 src');
64
+ expect(withProgressReport('bun test --timeout=0 src')).toBe('bun test --dots --timeout=0 src');
65
65
  expect(withProgressReport('../x/.runtime/bun test a')).toBe('../x/.runtime/bun test --dots a');
66
66
  });
67
67