@smoothbricks/nx-plugin 0.4.28 → 0.4.30

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (158) hide show
  1. package/README.md +277 -89
  2. package/dist/bin/smoo-nx-test-clocks.d.ts +3 -0
  3. package/dist/bin/smoo-nx-test-clocks.d.ts.map +1 -0
  4. package/dist/bin/smoo-nx-test-clocks.js +28 -0
  5. package/dist/bounded-test-policy.d.ts +8 -8
  6. package/dist/bounded-test-policy.d.ts.map +1 -1
  7. package/dist/bounded-test-policy.js +18 -30
  8. package/dist/cargo-closure-input.d.ts +23 -0
  9. package/dist/cargo-closure-input.d.ts.map +1 -1
  10. package/dist/cargo-closure-input.js +41 -9
  11. package/dist/cargo-source-hash.d.ts.map +1 -1
  12. package/dist/cargo-source-hash.js +42 -19
  13. package/dist/ensure-built-receipt.d.ts +40 -6
  14. package/dist/ensure-built-receipt.d.ts.map +1 -1
  15. package/dist/ensure-built-receipt.js +297 -159
  16. package/dist/ensure-built-witness.d.ts +65 -4
  17. package/dist/ensure-built-witness.d.ts.map +1 -1
  18. package/dist/ensure-built-witness.js +2237 -23
  19. package/dist/ensure-built.d.ts +99 -1
  20. package/dist/ensure-built.d.ts.map +1 -1
  21. package/dist/ensure-built.js +45 -56
  22. package/dist/executors/bounded-exec/cpu-pool.d.ts +22 -7
  23. package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -1
  24. package/dist/executors/bounded-exec/cpu-pool.js +51 -22
  25. package/dist/executors/bounded-exec/executor.d.ts +12 -3
  26. package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
  27. package/dist/executors/bounded-exec/executor.js +174 -51
  28. package/dist/executors/bounded-exec/no-space.d.ts +8 -0
  29. package/dist/executors/bounded-exec/no-space.d.ts.map +1 -0
  30. package/dist/executors/bounded-exec/no-space.js +31 -0
  31. package/dist/executors/bounded-exec/ram-temp.d.ts +20 -11
  32. package/dist/executors/bounded-exec/ram-temp.d.ts.map +1 -1
  33. package/dist/executors/bounded-exec/ram-temp.js +42 -17
  34. package/dist/executors/bounded-exec/verdict.d.ts +34 -4
  35. package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
  36. package/dist/executors/bounded-exec/verdict.js +54 -3
  37. package/dist/generators/bun-test-tracing/generator.js +2 -2
  38. package/dist/graph-input-contract.d.ts +102 -0
  39. package/dist/graph-input-contract.d.ts.map +1 -0
  40. package/dist/graph-input-contract.js +637 -0
  41. package/dist/index.d.ts +12 -5
  42. package/dist/index.d.ts.map +1 -1
  43. package/dist/index.js +17 -3
  44. package/dist/managed-files/ci-workflow.d.ts.map +1 -1
  45. package/dist/managed-files/ci-workflow.js +5 -2
  46. package/dist/managed-files/publish-workflow.d.ts +11 -1
  47. package/dist/managed-files/publish-workflow.d.ts.map +1 -1
  48. package/dist/owned-installation.d.ts +14 -0
  49. package/dist/owned-installation.d.ts.map +1 -0
  50. package/dist/owned-installation.js +324 -0
  51. package/dist/producer-receipts.d.ts +19 -0
  52. package/dist/producer-receipts.d.ts.map +1 -0
  53. package/dist/producer-receipts.js +926 -0
  54. package/dist/test-backstop.d.ts +9 -0
  55. package/dist/test-backstop.d.ts.map +1 -0
  56. package/dist/test-backstop.js +10 -0
  57. package/dist/test-clock-policy.d.ts +30 -0
  58. package/dist/test-clock-policy.d.ts.map +1 -0
  59. package/dist/test-clock-policy.js +686 -0
  60. package/dist/testing.d.ts +8 -0
  61. package/dist/testing.d.ts.map +1 -1
  62. package/dist/testing.js +66 -1
  63. package/managed/raw/patches/nx@23.2.1.patch +328 -50
  64. package/managed/raw/tooling/direnv/developer-links.ts +148 -29
  65. package/managed/raw/tooling/direnv/devenv.smoo.nix +7 -7
  66. package/managed/raw/tooling/direnv/github-actions-bootstrap.sh +14 -5
  67. package/managed/raw/tooling/direnv/linked-snapshots.ts +206 -40
  68. package/managed/raw/tooling/direnv/repo-path +15 -2
  69. package/managed/raw/tooling/direnv/setup-environment.ts +179 -115
  70. package/managed/templates/github/actions/cache-node-modules/action.yml +9 -0
  71. package/managed/templates/github/actions/setup-devenv/action.yml +63 -7
  72. package/nextest.toml +1 -1
  73. package/package.json +25 -9
  74. package/src/__tests__/cargo-closure-fixture.ts +113 -0
  75. package/src/__tests__/counted-cargo.ts +8 -25
  76. package/src/__tests__/directory-change.ts +27 -0
  77. package/src/__tests__/fixture-nx-env.ts +7 -0
  78. package/src/__tests__/owned-installation.ts +14 -0
  79. package/src/__tests__/past-tick.ts +12 -0
  80. package/src/bin/smoo-nx-bound-failures.test.ts +1 -1
  81. package/src/bin/smoo-nx-test-clocks.ts +29 -0
  82. package/src/bounded-test-policy.test.ts +36 -22
  83. package/src/bounded-test-policy.ts +18 -32
  84. package/src/bun-bootstrap.test.ts +1 -1
  85. package/src/cargo-closure-input.property.test.ts +50 -0
  86. package/src/cargo-closure-input.test.ts +73 -121
  87. package/src/cargo-closure-input.ts +47 -8
  88. package/src/cargo-closure-key.test.ts +239 -0
  89. package/src/cargo-resolve-cache.test.ts +168 -31
  90. package/src/cargo-source-hash.test.ts +78 -3
  91. package/src/cargo-source-hash.ts +48 -30
  92. package/src/cross-check-policy.test.ts +2 -2
  93. package/src/ensure-built-receipt.test.ts +111 -16
  94. package/src/ensure-built-receipt.ts +294 -149
  95. package/src/ensure-built-witness.test.ts +306 -13
  96. package/src/ensure-built-witness.ts +255 -24
  97. package/src/ensure-built.test.ts +228 -102
  98. package/src/ensure-built.ts +127 -66
  99. package/src/executors/bounded-exec/cpu-pool.test.ts +134 -32
  100. package/src/executors/bounded-exec/cpu-pool.ts +64 -20
  101. package/src/executors/bounded-exec/cpu-tokens.test.ts +2 -2
  102. package/src/executors/bounded-exec/disk-lease.test.ts +47 -13
  103. package/src/executors/bounded-exec/executor-host.test.ts +74 -52
  104. package/src/executors/bounded-exec/executor.test.ts +343 -97
  105. package/src/executors/bounded-exec/executor.ts +192 -56
  106. package/src/executors/bounded-exec/hang-diagnostics.test.ts +14 -8
  107. package/src/executors/bounded-exec/no-space.ts +40 -0
  108. package/src/executors/bounded-exec/progress.test.ts +1 -1
  109. package/src/executors/bounded-exec/ram-temp-host.test.ts +51 -21
  110. package/src/executors/bounded-exec/ram-temp.test.ts +92 -16
  111. package/src/executors/bounded-exec/ram-temp.ts +49 -20
  112. package/src/executors/bounded-exec/verdict.test.ts +127 -22
  113. package/src/executors/bounded-exec/verdict.ts +78 -3
  114. package/src/generators/bounded-test-targets/generator.test.ts +1 -3
  115. package/src/generators/bun-test-tracing/generator.test.ts +2 -2
  116. package/src/generators/bun-test-tracing/generator.ts +2 -2
  117. package/src/graph-input-contract.test.ts +379 -0
  118. package/src/graph-input-contract.ts +627 -0
  119. package/src/index.test.ts +3 -3
  120. package/src/index.ts +24 -3
  121. package/src/linked-snapshots.nx.test.ts +112 -30
  122. package/src/managed-files/ci-workflow.ts +5 -2
  123. package/src/managed-files/publish-workflow.ts +12 -1
  124. package/src/managed-files/repo-path.test.ts +44 -1
  125. package/src/managed-files/setup-devenv-free-disk.test.ts +95 -0
  126. package/src/nextest-code-bound.test.ts +23 -17
  127. package/src/nextest-extraction.test.ts +114 -4
  128. package/src/nextest-partitions.test.ts +7 -1
  129. package/src/nx-cache-restore-mtime.test.ts +2 -2
  130. package/src/nx-daemon-claim.test.ts +10 -8
  131. package/src/nx-daemon-idle-cpu.test.ts +0 -1
  132. package/src/nx-daemon-idle-policy.test.ts +61 -1
  133. package/src/nx-daemon-ignore-files.test.ts +2 -2
  134. package/src/nx-daemon-validity.test.ts +36 -22
  135. package/src/nx-dot-input-policy.test.ts +5 -5
  136. package/src/nx-files-archive-racy.test.ts +1 -1
  137. package/src/nx-incomplete-run-summary.test.ts +1 -1
  138. package/src/nx-plugin-load-errors.test.ts +2 -2
  139. package/src/nx-project-graph-errors.test.ts +70 -0
  140. package/src/nx-skipped-task-exit.test.ts +2 -2
  141. package/src/nx-store-resolution.test.ts +7 -25
  142. package/src/nx-task-output-lease.test.ts +3 -3
  143. package/src/nx-use-local.test.ts +7 -5
  144. package/src/owned-installation.ts +174 -0
  145. package/src/producer-receipts.test.ts +164 -0
  146. package/src/producer-receipts.ts +636 -0
  147. package/src/runtime-input.test.ts +1 -1
  148. package/src/serial-group-wiring.test.ts +4 -4
  149. package/src/task-results.test.ts +18 -16
  150. package/src/test-backstop.ts +11 -0
  151. package/src/test-clock-policy.test.ts +338 -0
  152. package/src/test-clock-policy.ts +708 -0
  153. package/src/test-timings.test.ts +2 -2
  154. package/src/testing.test.ts +5 -5
  155. package/src/testing.ts +67 -1
  156. package/src/typecheck-test-policy.test.ts +1 -1
  157. package/src/typescript-resolution.test.ts +76 -0
  158. package/test-timings.json +8 -0
@@ -1,12 +1,14 @@
1
1
  import { type ChildProcess, spawn } from 'node:child_process';
2
+ import { accessSync, constants } from 'node:fs';
2
3
  import { availableParallelism, userInfo } from 'node:os';
3
- import { isAbsolute, join, resolve as resolvePath } from 'node:path';
4
+ import { delimiter, isAbsolute, join, resolve as resolvePath } from 'node:path';
4
5
  import { workspaceDataDirectoryForWorkspace } from 'nx/src/utils/cache-directory.js';
5
6
 
6
7
  import {
7
8
  CPU_POOL_ENV,
8
9
  type CpuPool,
9
10
  createCpuPool,
11
+ type HostWaits,
10
12
  type InheritedCpuPool,
11
13
  readInheritedCpuPool,
12
14
  takeParentCpuTokens,
@@ -21,6 +23,7 @@ import {
21
23
  } from './cpu-tokens.js';
22
24
  import type { HangDiagnostics } from './hang-diagnostics.js';
23
25
  import { captureHangDiagnostics, HANG_CAPTURE_MS } from './hang-diagnostics.js';
26
+ import { createNoSpaceWatch } from './no-space.js';
24
27
  import { createRunProgress, withProgressReport } from './progress.js';
25
28
  import {
26
29
  COWSHED_EXECUTABLE_ENV,
@@ -36,11 +39,13 @@ import {
36
39
  type BoundedExecRecord,
37
40
  type BoundedExecTask,
38
41
  clearTaskDirectory,
42
+ describeNoSpace,
39
43
  type ExpiredBound,
40
44
  JUNIT_FILE,
41
45
  JUNIT_REPORT_ENV,
42
46
  judgeOperationalFailure,
43
47
  judgeRun,
48
+ type NoSpaceSighting,
44
49
  readJunitReport,
45
50
  taskDirectory,
46
51
  withJunitReport,
@@ -88,10 +93,10 @@ interface RunState {
88
93
  settled: boolean;
89
94
  expiry: ExpiredBound | null;
90
95
  forceKillNeeded: boolean;
91
- /** Set when a borrowed grant closed while the command ran, and the command was stopped as a lost grant. */
92
- grantLost: boolean;
93
96
  /** The sub-pool's close error when it did not close cleanly: the run fails. */
94
97
  poolCloseError: string | null;
98
+ /** The first output line that said ENOSPC, with the task's volume read when it arrived. */
99
+ noSpace: Promise<NoSpaceSighting> | null;
95
100
  }
96
101
 
97
102
  type GrantedTokens = Extract<TokenGrant, { granted: true }>;
@@ -101,7 +106,53 @@ export interface ProcessTreeKiller {
101
106
  }
102
107
 
103
108
  /** Where the command's TMPDIR comes from; `null` leaves the inherited one. */
104
- export type TempVolume = Pick<RamTempVolume, 'acquire' | 'release' | 'fullness' | 'reap'>;
109
+ export type TempVolume = Pick<RamTempVolume, 'acquire' | 'release' | 'fullness' | 'space' | 'reap'>;
110
+
111
+ /** Workspace-provided cowshed launchers, in the order a workspace that provides one names them. */
112
+ const WORKSPACE_COWSHED_LAUNCHERS = [
113
+ 'packages/cowshed/bin/cowshed',
114
+ 'node_modules/@smoothbricks/cowshed/bin/cowshed',
115
+ ] as const;
116
+
117
+ /**
118
+ * The cowshed a run's disk commands use, in precedence order: the target's named one, an
119
+ * explicit one inherited from the parent process, the installed one on PATH, and — only when
120
+ * PATH provides none — the launcher this workspace itself provides (its own cowshed package,
121
+ * or a linked one's bin). An environment with no cowshed anywhere keeps the bare name, so
122
+ * its spawn failure names the missing installed tool.
123
+ */
124
+ export function resolveCowshedExecutable(
125
+ root: string,
126
+ named: string | undefined,
127
+ inherited: string | undefined,
128
+ pathValue: string | undefined,
129
+ cwd: string,
130
+ exists: (path: string) => boolean = (path) => {
131
+ try {
132
+ accessSync(path, constants.X_OK);
133
+ return true;
134
+ } catch {
135
+ return false;
136
+ }
137
+ },
138
+ ): string | undefined {
139
+ // A target-named executable is workspace-relative or absolute against the workspace root.
140
+ // An inherited explicit keeps the coordinate the parent gave it, exactly as the host
141
+ // commands always spawned it: an absolute one verbatim, a relative or bare one for
142
+ // spawn's own resolution — retargeting it to the root would silently change the tool.
143
+ if (named !== undefined) return resolvePath(root, named);
144
+ if (inherited !== undefined) return inherited;
145
+ // An empty PATH component names the spawning process's current directory, as POSIX
146
+ // spawn resolution does — the executor never chdirs, so that is the caller's cwd,
147
+ // not the workspace root.
148
+ const onPath = (pathValue ?? '').split(delimiter).some((dir) => exists(join(dir === '' ? cwd : dir, 'cowshed')));
149
+ if (onPath) return undefined;
150
+ for (const candidate of WORKSPACE_COWSHED_LAUNCHERS) {
151
+ const launcher = join(root, candidate);
152
+ if (exists(launcher)) return launcher;
153
+ }
154
+ return undefined;
155
+ }
105
156
 
106
157
  export default function boundedExecExecutor(
107
158
  options: BoundedExecOptions,
@@ -119,7 +170,13 @@ export function runDefaultBoundedExec(
119
170
  // for its command alike. Nx resolves `{workspaceRoot}/…` in an option to a workspace-relative path, which the
120
171
  // command, starting in another directory, could not find: it gets the absolute one.
121
172
  const named = options.env?.[COWSHED_EXECUTABLE_ENV];
122
- const cowshed = named === undefined ? undefined : resolvePath(context.root, named);
173
+ const cowshed = resolveCowshedExecutable(
174
+ context.root,
175
+ named,
176
+ process.env[COWSHED_EXECUTABLE_ENV],
177
+ process.env.PATH,
178
+ process.cwd(),
179
+ );
123
180
  const run: BoundedExecOptions =
124
181
  cowshed === undefined ? options : { ...options, env: { ...options.env, [COWSHED_EXECUTABLE_ENV]: cowshed } };
125
182
  // Only macOS has `hdiutil ram://`; elsewhere the command keeps the inherited TMPDIR.
@@ -167,8 +224,9 @@ export function recordContext(context: NxTaskContext): BoundedExecContext {
167
224
  *
168
225
  * `inherited` is the parent's CPU pool (`cpu-pool.ts`) that lends this run its tokens. The run asks that pool first
169
226
  * and the host budget only when the pool refuses. Whatever it is granted, the run serves its own sub-pool to its
170
- * descendants, closed before the grant returns. A borrowed grant that closes while the command runs stops the command
171
- * as a lost grant, which is a failure.
227
+ * descendants, closed before the grant returns. A grant whose connection closes while the command runs (a gateway
228
+ * restart, or the lending parent's exit) ends its accounting, never the command: the command runs on over budget to
229
+ * its own exit, and the run says so.
172
230
  */
173
231
  export async function runBoundedExec(
174
232
  options: BoundedExecOptions,
@@ -194,8 +252,8 @@ export async function runBoundedExec(
194
252
  settled: false,
195
253
  expiry: null,
196
254
  forceKillNeeded: false,
197
- grantLost: false,
198
255
  poolCloseError: null,
256
+ noSpace: null,
199
257
  };
200
258
  const appendStdout = (chunk: Buffer | string): void => {
201
259
  const text = chunk.toString();
@@ -261,13 +319,13 @@ export async function runBoundedExec(
261
319
  const cores = availableParallelism();
262
320
  const demand = runnerDemand(requested, options.parallelism, cores, process.env, options.env);
263
321
  const asked = Date.now();
264
- // A run a parent's CPU pool lent tokens to takes them there. Only an absent pool leaves the host budget to ask.
265
- // A pool that answered and refused still holds tokens of its own: asking the host then would hold them while waiting
266
- // on another budget, so the run fails instead.
322
+ // A run a parent's CPU pool serves takes its tokens there. Only an absent pool leaves the host budget to ask.
323
+ // A pool that answered and refused is a live budget of its own -- a lent grant, or a gate's or a resident's
324
+ // relay to the host -- so asking the host beside it would count the run twice; the run fails instead.
267
325
  let grant: GrantedTokens | null = null;
268
326
  let parentRefusal: string | null = null;
269
327
  if (inherited !== null) {
270
- const fromParent = await takeParentCpuTokens(inherited, demand.want, cores);
328
+ const fromParent = await takeParentCpuTokens(inherited, demand.want);
271
329
  if (fromParent.granted) {
272
330
  grant = fromParent;
273
331
  } else if (fromParent.cause === 'absent') {
@@ -318,13 +376,14 @@ export async function runBoundedExec(
318
376
  return cpuReleased;
319
377
  };
320
378
  // Records the run's verdict when it has a task to record as. An `expiry` is set only when a bound ended the run.
321
- // An `operational` run failed on its CPU budget (a lost grant, or a sub-pool that did not close): an ordinary
322
- // failure, whatever bound also fired, so it is never judged as a bound.
379
+ // An `operational` run failed on its CPU budget (a sub-pool that did not close): an ordinary failure, whatever
380
+ // bound also fired, so it is never judged as a bound.
323
381
  const recordRun = async (
324
382
  exitCode: number,
325
383
  expiry: ExpiredBound | null,
326
384
  elapsedMs: number,
327
385
  operational: boolean,
386
+ noSpace: NoSpaceSighting | null,
328
387
  ): Promise<void> => {
329
388
  if (context.record === undefined) {
330
389
  return;
@@ -333,7 +392,7 @@ export async function runBoundedExec(
333
392
  const report = await readJunitReport(recordDirectory);
334
393
  const verdict = operational
335
394
  ? judgeOperationalFailure(exitCode, report)
336
- : judgeRun({ exitCode, elapsedMs, expiry, report });
395
+ : judgeRun({ exitCode, elapsedMs, expiry, report, noSpace });
337
396
  if (verdict.outcome === 'bound' && verdict.bound === 'test') {
338
397
  appendStderr(`Every failing test failed only on its per-test timeout: ${verdict.tests.join(', ')}\n`);
339
398
  }
@@ -344,7 +403,7 @@ export async function runBoundedExec(
344
403
  appendStderr(message);
345
404
  await releaseCpu();
346
405
  await releaseLease();
347
- await recordRun(1, null, Date.now() - asked, false);
406
+ await recordRun(1, null, Date.now() - asked, false, null);
348
407
  return { success: false, terminalOutput: outputChunks.join('') };
349
408
  };
350
409
  if (parentRefusal !== null) {
@@ -353,10 +412,26 @@ export async function runBoundedExec(
353
412
  );
354
413
  }
355
414
 
415
+ // Descendant work queued for host CPU tokens, as its pool hears it (`host-wait`): while any is held the idle clock
416
+ // is stopped, because a queued job is not an idle one (specs/cowshed/05_gateway.md, "The bounds start at the
417
+ // grant"). The clock it stops is made with the command below; a wait held before then is counted and honoured there.
418
+ let heldWaits = 0;
419
+ let idleClock: { stop(): void; restart(): void } | null = null;
420
+ const hostWaits: HostWaits = {
421
+ begin: () => {
422
+ heldWaits += 1;
423
+ if (heldWaits === 1) idleClock?.stop();
424
+ },
425
+ end: () => {
426
+ heldWaits -= 1;
427
+ if (heldWaits === 0) idleClock?.restart();
428
+ },
429
+ };
430
+
356
431
  // A granted run serves a sub-pool of its own grant, so its descendants draw from that and never from above it.
357
432
  // A pool that cannot be set up refuses the run: no command runs unbudgeted.
358
433
  if (grant !== null) {
359
- const created = await createCpuPool(grant.tokens, grant.closed);
434
+ const created = await createCpuPool(grant.tokens, grant.closed, hostWaits);
360
435
  if (!created.ok) {
361
436
  return refuse(`cowshed: cpu pool unavailable: ${created.error}; refusing to run the command\n`);
362
437
  }
@@ -393,6 +468,13 @@ export async function runBoundedExec(
393
468
 
394
469
  // Progress is a finished test, not output: a runner's SLOW warnings and a hung test's heartbeat keep printing.
395
470
  const progress = createRunProgress();
471
+ // A write that found no space is said once, in the output, by whatever made it; the volume is read when it is said,
472
+ // since the writer usually removes what it half-wrote before it exits and the volume no longer looks full by then.
473
+ const noSpace = createNoSpaceWatch();
474
+ const sight = async (line: string): Promise<NoSpaceSighting> => {
475
+ const held = lease;
476
+ return { line, volume: held === null ? null : ((await tempVolume?.space(held)) ?? null) };
477
+ };
396
478
  let idleTimer: NodeJS.Timeout | undefined;
397
479
  let captureTimer: NodeJS.Timeout | undefined;
398
480
  let idleCapture: { expiresAtMs: number; controller: AbortController; result: IdleDiagnosticResult } | undefined;
@@ -400,14 +482,17 @@ export async function runBoundedExec(
400
482
  clearTimeout(captureTimer);
401
483
  idleCapture?.controller.abort();
402
484
  };
403
- const armIdleTimer = (): void => {
404
- if (idleTimeoutMs === undefined || state.settled || state.expiry !== null) {
405
- return;
406
- }
485
+ // What is left of the current no-progress interval. A finished test sets it back to `idleTimeoutMs`; a held host
486
+ // wait keeps it unspent: the clock is stopped while descendant work is queued for CPU tokens, and runs on from where
487
+ // it stopped once the last wait ends. `queuedMs` is the queued time this interval has not counted, for the report.
488
+ let idleLeftMs = idleTimeoutMs ?? 0;
489
+ let queuedMs = 0;
490
+ let queuedSinceMs: number | null = null;
491
+ const scheduleIdle = (limitMs: number, leftMs: number): void => {
407
492
  clearTimeout(idleTimer);
408
493
  cancelCapture();
409
494
  const capture: NonNullable<typeof idleCapture> = {
410
- expiresAtMs: Date.now() + idleTimeoutMs,
495
+ expiresAtMs: Date.now() + leftMs,
411
496
  controller: new AbortController(),
412
497
  result: { kind: 'pending', startedAtMs: null },
413
498
  };
@@ -445,12 +530,51 @@ export async function runBoundedExec(
445
530
  },
446
531
  );
447
532
  },
448
- Math.max(0, idleTimeoutMs - HANG_CAPTURE_MS),
449
- );
450
- idleTimer = setTimeout(
451
- () => expire({ kind: 'idle', limitMs: idleTimeoutMs, tests: progress.inFlight() }),
452
- idleTimeoutMs,
533
+ Math.max(0, leftMs - HANG_CAPTURE_MS),
453
534
  );
535
+ idleTimer = setTimeout(() => expire({ kind: 'idle', limitMs, tests: progress.inFlight() }, queuedMs), leftMs);
536
+ };
537
+ const armIdleTimer = (): void => {
538
+ if (idleTimeoutMs === undefined || state.settled || state.expiry !== null) {
539
+ return;
540
+ }
541
+ idleLeftMs = idleTimeoutMs;
542
+ queuedMs = 0;
543
+ if (queuedSinceMs !== null) {
544
+ queuedSinceMs = Date.now();
545
+ return;
546
+ }
547
+ scheduleIdle(idleTimeoutMs, idleLeftMs);
548
+ };
549
+ idleClock = {
550
+ stop: () => {
551
+ if (idleTimeoutMs === undefined || state.settled || state.expiry !== null || queuedSinceMs !== null) {
552
+ return;
553
+ }
554
+ const now = Date.now();
555
+ queuedSinceMs = now;
556
+ if (idleCapture !== undefined) {
557
+ idleLeftMs = Math.max(0, idleCapture.expiresAtMs - now);
558
+ }
559
+ clearTimeout(idleTimer);
560
+ cancelCapture();
561
+ idleCapture = undefined;
562
+ appendStderr(
563
+ `bounded-exec: idle clock stopped: a descendant's work is queued for host CPU tokens; ${idleLeftMs}ms of idleTimeoutMs=${idleTimeoutMs} left\n`,
564
+ );
565
+ },
566
+ restart: () => {
567
+ if (idleTimeoutMs === undefined || state.settled || state.expiry !== null || queuedSinceMs === null) {
568
+ return;
569
+ }
570
+ const waitedMs = Date.now() - queuedSinceMs;
571
+ queuedMs += waitedMs;
572
+ queuedSinceMs = null;
573
+ appendStderr(
574
+ `bounded-exec: idle clock restarted: no descendant's work is queued for host CPU tokens after ${waitedMs}ms; ${idleLeftMs}ms of idleTimeoutMs=${idleTimeoutMs} left\n`,
575
+ );
576
+ scheduleIdle(idleTimeoutMs, idleLeftMs);
577
+ },
454
578
  };
455
579
 
456
580
  // Only the CHILD's own output is read for progress. The diagnostics below go
@@ -459,11 +583,15 @@ export async function runBoundedExec(
459
583
  child.stdout?.on('data', (chunk: Buffer | string) => {
460
584
  const text = chunk.toString();
461
585
  if (progress.scan('stdout', text)) armIdleTimer();
586
+ const said = noSpace.scan('stdout', text);
587
+ if (said !== null) state.noSpace = sight(said);
462
588
  appendStdout(text);
463
589
  });
464
590
  child.stderr?.on('data', (chunk: Buffer | string) => {
465
591
  const text = chunk.toString();
466
592
  if (progress.scan('stderr', text)) armIdleTimer();
593
+ const said = noSpace.scan('stderr', text);
594
+ if (said !== null) state.noSpace = sight(said);
467
595
  appendStderr(text);
468
596
  });
469
597
 
@@ -522,8 +650,9 @@ export async function runBoundedExec(
522
650
  // the reader unable to tell a wedged toolchain from a loaded machine — the
523
651
  // two have opposite fixes.
524
652
  let termination: Promise<void> = Promise.resolve();
525
- const expire = (bound: ExpiredBound): void => {
526
- if (state.settled || state.expiry !== null || state.grantLost) {
653
+ // `queuedMs` is the time the idle interval did not count because descendant work was queued for host CPU tokens.
654
+ const expire = (bound: ExpiredBound, queuedMs = 0): void => {
655
+ if (state.settled || state.expiry !== null) {
527
656
  return;
528
657
  }
529
658
  state.expiry = bound;
@@ -532,7 +661,9 @@ export async function runBoundedExec(
532
661
  const elapsedMs = Date.now() - startedAt;
533
662
  appendStderr(
534
663
  bound.kind === 'idle'
535
- ? `\nCommand made no progress: no test finished for ${bound.limitMs}ms (idleTimeoutMs=${bound.limitMs}) after ${elapsedMs}ms of runtime; ${
664
+ ? `\nCommand made no progress: no test finished for ${bound.limitMs}ms (idleTimeoutMs=${bound.limitMs})${
665
+ queuedMs > 0 ? `, not counting ${queuedMs}ms queued for host CPU tokens,` : ''
666
+ } after ${elapsedMs}ms of runtime; ${
536
667
  bound.tests.length > 0
537
668
  ? `still running: ${bound.tests.join(', ')}`
538
669
  : 'the runner named no test still running'
@@ -601,25 +732,23 @@ export async function runBoundedExec(
601
732
  const totalTimer =
602
733
  timeoutMs === undefined ? undefined : setTimeout(() => expire({ kind: 'total', limitMs: timeoutMs }), timeoutMs);
603
734
  armIdleTimer();
735
+ // A wait its pool heard before the command started stops the clock it was not yet there to stop.
736
+ if (heldWaits > 0) idleClock.stop();
604
737
 
605
- // A borrowed grant ends with the ancestor that lent it, and the command runs on tokens that ancestor takes back
606
- // then. Stopping the command is a lost grant: a failure, not a bound. Our own release after the command settled
607
- // is not a loss; `cpuReleased` marks it.
608
- const onGrantLost = (): void => {
609
- if (state.settled || state.grantLost || cpuReleased !== null) {
610
- return;
611
- }
612
- state.grantLost = true;
613
- cancelCapture();
614
- appendStderr(
615
- `cowshed: cpu-tokens grant lost: its parent pool closed while the command ran; stopping it: ${command}\n`,
616
- );
617
- // A bound may already be terminating the still-running command. The loss remains an ordinary
618
- // operational failure, but that existing termination owns the signals and grace.
619
- if (state.expiry === null) termination = terminate();
620
- };
621
- if (grant?.inherited) {
622
- void grant.closed.then(onGrantLost);
738
+ // A grant's connection is a ledger entry, not the command's owner (05_gateway.md "Host CPU budget": a gateway
739
+ // restart forgets every grant, and what runs under it runs on over budget until it finishes). Its close, while the
740
+ // command runs, is said and changes nothing else; only a bound or a signal stops the command. Our own release after
741
+ // the command settled is not a loss; `cpuReleased` marks it.
742
+ if (grant !== null) {
743
+ const held = grant;
744
+ void held.closed.then(() => {
745
+ if (state.settled || cpuReleased !== null) {
746
+ return;
747
+ }
748
+ appendStderr(
749
+ `cowshed: cpu-tokens grant ended: the ${held.inherited ? 'parent pool' : 'gateway'} closed its ${held.tokens}-token grant while the command ran; it runs on over budget until it ends: ${command}\n`,
750
+ );
751
+ });
623
752
  }
624
753
 
625
754
  const { code: exitCode, signal: exitSignal } = await exitPromise;
@@ -641,17 +770,24 @@ export async function runBoundedExec(
641
770
  appendStderr(`Timed out command exited after graceful termination: ${command}\n`);
642
771
  }
643
772
 
644
- // A test that dies of ENOSPC rarely says where it was writing; the volume can.
645
- const full = code !== 0 && lease !== null ? await tempVolume?.fullness(lease) : null;
646
- if (full) {
647
- appendStderr(`${describeRamTempError(full)}\n`);
773
+ // ENOSPC said in the output names the writer's volume as it stood then. Without it, a volume still (nearly) full now
774
+ // is the likely cause, and the volume is named.
775
+ const seen = state.noSpace === null ? null : await state.noSpace;
776
+ if (seen !== null && (code !== 0 || state.expiry !== null)) {
777
+ const writer = context.record?.task.id;
778
+ appendStderr(`${writer === undefined ? '' : `${writer}: `}${describeNoSpace(seen)}\n`);
779
+ } else {
780
+ const full = code !== 0 && lease !== null ? await tempVolume?.fullness(lease) : null;
781
+ if (full) {
782
+ appendStderr(`${describeRamTempError(full)}\n`);
783
+ }
648
784
  }
649
785
  await releaseLease();
650
786
 
651
- // A run whose grant was lost, or whose sub-pool did not close cleanly, fails even when the command exited 0, and
652
- // whatever bound also fired: its verdict is an ordinary failure, never a bound.
653
- const operationalFailure = state.grantLost || state.poolCloseError !== null;
654
- await recordRun(code, state.expiry, Date.now() - startedAt, operationalFailure);
787
+ // A run whose sub-pool did not close cleanly fails even when the command exited 0, and whatever bound also fired:
788
+ // its verdict is an ordinary failure, never a bound.
789
+ const operationalFailure = state.poolCloseError !== null;
790
+ await recordRun(code, state.expiry, Date.now() - startedAt, operationalFailure, seen);
655
791
 
656
792
  return {
657
793
  success: !operationalFailure && state.expiry === null && code === 0,
@@ -1,7 +1,8 @@
1
- import { describe, expect, it } from 'bun:test';
1
+ import { describe, expect, it, jest } from 'bun:test';
2
2
  import { mkdtemp, rm, writeFile } from 'node:fs/promises';
3
3
  import { tmpdir } from 'node:os';
4
4
  import { join } from 'node:path';
5
+ import { BOUNDED_TEST_IDLE_TIMEOUT_MS } from '../../bounded-test-policy.js';
5
6
  import {
6
7
  captureHangDiagnostics,
7
8
  type HangCaptureRequest,
@@ -23,8 +24,8 @@ const exited = (stdout: string, code = 0, stderr = ''): HangProbeResult => ({ st
23
24
  const request = (): HangCaptureRequest => ({
24
25
  pid: ROOT,
25
26
  pgid: ROOT,
26
- expiresAtMs: Date.now() + 5_000,
27
- deadlineMs: 5_000,
27
+ expiresAtMs: Date.now() + BOUNDED_TEST_IDLE_TIMEOUT_MS,
28
+ deadlineMs: BOUNDED_TEST_IDLE_TIMEOUT_MS,
28
29
  });
29
30
 
30
31
  function scriptedHost(results: readonly HangProbeResult[]) {
@@ -140,6 +141,7 @@ describe('pre-expiry hang diagnostics', () => {
140
141
  return new Promise<HangProbeResult>(() => {});
141
142
  },
142
143
  };
144
+ // The probe is held off by construction, so the 1 ms deadline timer is the only event that can end this capture.
143
145
  const capture = await captureHangDiagnostics({ ...request(), deadlineMs: 1 }, unresponsive);
144
146
  expect(capture.outcome).toBe('incomplete');
145
147
  expect(capture.text).toContain('1ms deadline reached during ps before');
@@ -228,12 +230,11 @@ describe('pre-expiry hang diagnostics', () => {
228
230
  let descendant: number | undefined;
229
231
  const failures: unknown[] = [];
230
232
  try {
231
- const reader = child.stdout
232
- .pipeThrough(new TextDecoderStream(), { signal: AbortSignal.timeout(10_000) })
233
- .getReader();
233
+ const reader = child.stdout.pipeThrough(new TextDecoderStream()).getReader();
234
234
  let text = '';
235
235
  try {
236
- // Real OS-pipe readiness cannot advance a fake test clock; abort only bounds a missing line.
236
+ // The first complete line is the readiness event; a child that exits first ends the stream instead. No timer
237
+ // ends this loop.
237
238
  for (;;) {
238
239
  const chunk = await reader.read();
239
240
  if (chunk.done) throw new Error('child stdout ended before a complete PID line');
@@ -252,7 +253,12 @@ describe('pre-expiry hang diagnostics', () => {
252
253
  Number.isSafeInteger(descendant) && descendant > 0,
253
254
  `expected a tagged PID line, observed ${JSON.stringify(line)}`,
254
255
  ).toBe(true);
255
- const capture = await captureHangDiagnostics({ ...request(), pid: child.pid, pgid: null });
256
+ // The capture budget (`HANG_CAPTURE_MS`) and probe limits are product constants this test cannot set, so the
257
+ // clock is stopped while the real `ps` and `lsof` run: each ends when it exits; no timer cuts it short.
258
+ jest.useFakeTimers();
259
+ const capture = await captureHangDiagnostics({ ...request(), pid: child.pid, pgid: null }).finally(() =>
260
+ jest.useRealTimers(),
261
+ );
256
262
  expect(capture.outcome).toBe('holders');
257
263
  expect(capture.text).toContain(String(child.pid));
258
264
  expect(capture.text).toContain(String(descendant));
@@ -0,0 +1,40 @@
1
+ import type { OutputStream } from './progress.js';
2
+
3
+ /**
4
+ * The operating system's own words for ENOSPC: the errno name Node and Bun print (`ENOSPC: no space left on device,
5
+ * write`), and `strerror(28)`, which Rust (`No space left on device (os error 28)`), Go and C print. Nothing else in a
6
+ * command's output says the disk was full, and a write that failed on it is otherwise reported as whatever its caller
7
+ * was doing: a failed test, an extraction error.
8
+ */
9
+ const NO_SPACE = /\bENOSPC\b|No space left on device/;
10
+ const ANSI = new RegExp(`${String.fromCharCode(27)}\\[[0-9;]*[A-Za-z]`, 'g');
11
+
12
+ /** Watches a command's output for the first line that says a write found no space. */
13
+ export interface NoSpaceWatch {
14
+ /** One chunk of one output stream: the first matching line, once, as printed; `null` before and after it. */
15
+ scan(stream: OutputStream, chunk: string): string | null;
16
+ }
17
+
18
+ export function createNoSpaceWatch(): NoSpaceWatch {
19
+ const partial: Record<OutputStream, string> = { stdout: '', stderr: '' };
20
+ let seen = false;
21
+ return {
22
+ scan(stream, chunk) {
23
+ if (seen) {
24
+ return null;
25
+ }
26
+ const lines = (partial[stream] + chunk).split('\n');
27
+ // The unfinished last line is kept for the next chunk, and read now too: a command that dies mid-line still
28
+ // printed the words.
29
+ partial[stream] = lines.at(-1) ?? '';
30
+ for (const line of lines) {
31
+ const text = line.replace(ANSI, '').trim();
32
+ if (NO_SPACE.test(text)) {
33
+ seen = true;
34
+ return text;
35
+ }
36
+ }
37
+ return null;
38
+ },
39
+ };
40
+ }
@@ -61,7 +61,7 @@ describe('run progress', () => {
61
61
 
62
62
  describe('progress report', () => {
63
63
  it('asks a lone bun test for dots, keeping the runtime the target named', () => {
64
- expect(withProgressReport('bun test --timeout=30000 src')).toBe('bun test --dots --timeout=30000 src');
64
+ expect(withProgressReport('bun test --timeout=0 src')).toBe('bun test --dots --timeout=0 src');
65
65
  expect(withProgressReport('../x/.runtime/bun test a')).toBe('../x/.runtime/bun test --dots a');
66
66
  });
67
67