@smoothbricks/nx-plugin 0.7.17 → 0.7.19

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (199) hide show
  1. package/README.md +315 -114
  2. package/dist/.build-inputs.json +47 -38
  3. package/dist/built-output-packages.js +1 -1
  4. package/dist/bun-lock-scope.d.ts +128 -20
  5. package/dist/bun-lock-scope.d.ts.map +1 -1
  6. package/dist/bun-lock-scope.js +267 -44
  7. package/dist/cargo-closure-input.d.ts +8 -6
  8. package/dist/cargo-closure-input.d.ts.map +1 -1
  9. package/dist/cargo-closure-input.js +15 -12
  10. package/dist/cargo-source-hash.d.ts +26 -25
  11. package/dist/cargo-source-hash.d.ts.map +1 -1
  12. package/dist/cargo-source-hash.js +84 -105
  13. package/dist/cargo-toolchain-paths.d.ts +2 -2
  14. package/dist/cargo-toolchain-paths.js +2 -2
  15. package/dist/cargo-workspace.d.ts +20 -2
  16. package/dist/cargo-workspace.d.ts.map +1 -1
  17. package/dist/cargo-workspace.js +43 -22
  18. package/dist/ensure-built-receipt.d.ts +3 -2
  19. package/dist/ensure-built-receipt.d.ts.map +1 -1
  20. package/dist/ensure-built-receipt.js +3 -2
  21. package/dist/eslint-config-closure.d.ts +8 -3
  22. package/dist/eslint-config-closure.d.ts.map +1 -1
  23. package/dist/eslint-config-closure.js +24 -5
  24. package/dist/executors/bounded-exec/admission-queue.d.ts +3 -1
  25. package/dist/executors/bounded-exec/admission-queue.d.ts.map +1 -1
  26. package/dist/executors/bounded-exec/admission-queue.js +9 -10
  27. package/dist/executors/bounded-exec/awake-clock.d.ts +5 -0
  28. package/dist/executors/bounded-exec/awake-clock.d.ts.map +1 -0
  29. package/dist/executors/bounded-exec/awake-clock.js +22 -0
  30. package/dist/executors/bounded-exec/bounds.d.ts +19 -6
  31. package/dist/executors/bounded-exec/bounds.d.ts.map +1 -1
  32. package/dist/executors/bounded-exec/bounds.js +22 -15
  33. package/dist/executors/bounded-exec/case-gate.d.ts +114 -0
  34. package/dist/executors/bounded-exec/case-gate.d.ts.map +1 -0
  35. package/dist/executors/bounded-exec/case-gate.js +342 -0
  36. package/dist/executors/bounded-exec/cpu-pool-thread.d.ts +1 -0
  37. package/dist/executors/bounded-exec/cpu-pool-thread.d.ts.map +1 -1
  38. package/dist/executors/bounded-exec/cpu-pool-thread.js +2 -2
  39. package/dist/executors/bounded-exec/cpu-pool.d.ts +28 -1
  40. package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -1
  41. package/dist/executors/bounded-exec/cpu-pool.js +102 -16
  42. package/dist/executors/bounded-exec/cpu-rise.d.ts +73 -0
  43. package/dist/executors/bounded-exec/cpu-rise.d.ts.map +1 -0
  44. package/dist/executors/bounded-exec/cpu-rise.js +111 -0
  45. package/dist/executors/bounded-exec/endpoint-life.d.ts +146 -0
  46. package/dist/executors/bounded-exec/endpoint-life.d.ts.map +1 -0
  47. package/dist/executors/bounded-exec/endpoint-life.js +228 -0
  48. package/dist/executors/bounded-exec/executor.d.ts +1 -1
  49. package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
  50. package/dist/executors/bounded-exec/executor.js +256 -86
  51. package/dist/executors/bounded-exec/gateway-lease.d.ts +1 -1
  52. package/dist/executors/bounded-exec/gateway-lease.d.ts.map +1 -1
  53. package/dist/executors/bounded-exec/gateway-lease.js +8 -1
  54. package/dist/executors/bounded-exec/go-run-summary.d.ts +51 -0
  55. package/dist/executors/bounded-exec/go-run-summary.d.ts.map +1 -0
  56. package/dist/executors/bounded-exec/go-run-summary.js +98 -0
  57. package/dist/executors/bounded-exec/hang-diagnostics.d.ts +11 -3
  58. package/dist/executors/bounded-exec/hang-diagnostics.d.ts.map +1 -1
  59. package/dist/executors/bounded-exec/hang-diagnostics.js +125 -47
  60. package/dist/executors/bounded-exec/lease-wait.d.ts +1 -0
  61. package/dist/executors/bounded-exec/lease-wait.d.ts.map +1 -1
  62. package/dist/executors/bounded-exec/lease-wait.js +1 -0
  63. package/dist/executors/bounded-exec/mbx-pool.d.ts +42 -7
  64. package/dist/executors/bounded-exec/mbx-pool.d.ts.map +1 -1
  65. package/dist/executors/bounded-exec/mbx-pool.js +45 -8
  66. package/dist/executors/bounded-exec/mbx-reservations.d.ts +12 -6
  67. package/dist/executors/bounded-exec/mbx-reservations.d.ts.map +1 -1
  68. package/dist/executors/bounded-exec/mbx-reservations.js +190 -284
  69. package/dist/executors/bounded-exec/parent-grant.d.ts +2 -0
  70. package/dist/executors/bounded-exec/parent-grant.d.ts.map +1 -1
  71. package/dist/executors/bounded-exec/parent-grant.js +1 -0
  72. package/dist/executors/bounded-exec/pool-admission.d.ts +29 -3
  73. package/dist/executors/bounded-exec/pool-admission.d.ts.map +1 -1
  74. package/dist/executors/bounded-exec/pool-admission.js +47 -1
  75. package/dist/executors/bounded-exec/progress-channel.d.ts +38 -0
  76. package/dist/executors/bounded-exec/progress-channel.d.ts.map +1 -0
  77. package/dist/executors/bounded-exec/progress-channel.js +113 -0
  78. package/dist/executors/bounded-exec/progress.d.ts +5 -1
  79. package/dist/executors/bounded-exec/progress.d.ts.map +1 -1
  80. package/dist/executors/bounded-exec/progress.js +31 -1
  81. package/dist/executors/bounded-exec/ram-temp.d.ts +24 -1
  82. package/dist/executors/bounded-exec/ram-temp.d.ts.map +1 -1
  83. package/dist/executors/bounded-exec/ram-temp.js +123 -16
  84. package/dist/executors/bounded-exec/reservation-connection.d.ts +90 -0
  85. package/dist/executors/bounded-exec/reservation-connection.d.ts.map +1 -0
  86. package/dist/executors/bounded-exec/reservation-connection.js +248 -0
  87. package/dist/executors/bounded-exec/run-cpu-pool.d.ts.map +1 -1
  88. package/dist/executors/bounded-exec/run-cpu-pool.js +3 -0
  89. package/dist/executors/bounded-exec/runner-sizing.d.ts +9 -5
  90. package/dist/executors/bounded-exec/runner-sizing.d.ts.map +1 -1
  91. package/dist/executors/bounded-exec/runner-sizing.js +13 -9
  92. package/dist/executors/bounded-exec/schema.d.ts +7 -2
  93. package/dist/executors/bounded-exec/schema.d.ts.map +1 -1
  94. package/dist/executors/bounded-exec/schema.js +17 -1
  95. package/dist/executors/bounded-exec/suite-minimum.d.ts +21 -0
  96. package/dist/executors/bounded-exec/suite-minimum.d.ts.map +1 -0
  97. package/dist/executors/bounded-exec/suite-minimum.js +85 -0
  98. package/dist/executors/bounded-exec/test-result-writer.d.ts +2 -0
  99. package/dist/executors/bounded-exec/test-result-writer.d.ts.map +1 -1
  100. package/dist/executors/bounded-exec/test-result-writer.js +1 -0
  101. package/dist/executors/bounded-exec/test-results.d.ts +5 -0
  102. package/dist/executors/bounded-exec/test-results.d.ts.map +1 -1
  103. package/dist/executors/bounded-exec/test-results.js +2 -2
  104. package/dist/executors/bounded-exec/verdict.d.ts +39 -10
  105. package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
  106. package/dist/executors/bounded-exec/verdict.js +60 -14
  107. package/dist/executors/cargo-zigbuild/executor.d.ts +10 -8
  108. package/dist/executors/cargo-zigbuild/executor.d.ts.map +1 -1
  109. package/dist/executors/cargo-zigbuild/executor.js +32 -19
  110. package/dist/index.d.ts.map +1 -1
  111. package/dist/index.js +294 -152
  112. package/dist/managed-files/ci-workflow.d.ts +4 -1
  113. package/dist/managed-files/ci-workflow.d.ts.map +1 -1
  114. package/dist/managed-files/ci-workflow.js +9 -14
  115. package/dist/managed-files/files.d.ts +3 -3
  116. package/dist/managed-files/files.js +3 -3
  117. package/dist/managed-files/generator.d.ts +5 -1
  118. package/dist/managed-files/generator.d.ts.map +1 -1
  119. package/dist/managed-files/generator.js +22 -7
  120. package/dist/managed-files/render-memo.d.ts +13 -0
  121. package/dist/managed-files/render-memo.d.ts.map +1 -0
  122. package/dist/managed-files/render-memo.js +94 -0
  123. package/dist/task-results.js +2 -1
  124. package/dist/test-program.d.ts +11 -6
  125. package/dist/test-program.d.ts.map +1 -1
  126. package/dist/test-program.js +39 -35
  127. package/dist/tool-bins.d.ts +12 -0
  128. package/dist/tool-bins.d.ts.map +1 -1
  129. package/dist/tool-bins.js +31 -6
  130. package/dist/typecheck-test-policy.d.ts +10 -1
  131. package/dist/typecheck-test-policy.d.ts.map +1 -1
  132. package/dist/typecheck-test-policy.js +40 -24
  133. package/dist/workspace-config-policy.d.ts.map +1 -1
  134. package/dist/workspace-config-policy.js +135 -42
  135. package/dist/workspace-index.d.ts +14 -1
  136. package/dist/workspace-index.d.ts.map +1 -1
  137. package/dist/workspace-index.js +36 -4
  138. package/executors.json +2 -0
  139. package/managed/raw/envrc +24 -0
  140. package/managed/raw/patches/nx@24.0.3.patch +1851 -221
  141. package/managed/raw/patches/ttsc@0.37.1.patch +1055 -0
  142. package/managed/raw/tooling/direnv/developer-links.ts +122 -14
  143. package/managed/raw/tooling/direnv/devenv.smoo.nix +20 -19
  144. package/managed/templates/github/actions/save-nix-devenv/action.yml +19 -1
  145. package/managed/templates/github/actions/setup-devenv/action.yml +2 -0
  146. package/package.json +68 -11
  147. package/runtime/nx-publish/.build-inputs.json +2 -2
  148. package/runtime/nx-publish/cache_adapter.js +293 -3
  149. package/runtime/nx-publish/nx_publish.js +36 -14
  150. package/src/built-output-packages.ts +1 -1
  151. package/src/bun-lock-scope.ts +293 -40
  152. package/src/cargo-closure-input.ts +20 -12
  153. package/src/cargo-source-hash.ts +119 -145
  154. package/src/cargo-toolchain-paths.ts +2 -2
  155. package/src/cargo-workspace.ts +59 -21
  156. package/src/ensure-built-receipt.ts +3 -2
  157. package/src/eslint-config-closure.ts +27 -6
  158. package/src/executors/bounded-exec/admission-queue.ts +11 -10
  159. package/src/executors/bounded-exec/awake-clock.ts +20 -0
  160. package/src/executors/bounded-exec/bounds.ts +41 -19
  161. package/src/executors/bounded-exec/case-gate.ts +365 -0
  162. package/src/executors/bounded-exec/cpu-pool-thread.ts +3 -3
  163. package/src/executors/bounded-exec/cpu-pool.ts +139 -12
  164. package/src/executors/bounded-exec/cpu-rise.ts +156 -0
  165. package/src/executors/bounded-exec/endpoint-life.ts +279 -0
  166. package/src/executors/bounded-exec/executor.ts +341 -123
  167. package/src/executors/bounded-exec/gateway-lease.ts +6 -0
  168. package/src/executors/bounded-exec/go-run-summary.ts +137 -0
  169. package/src/executors/bounded-exec/hang-diagnostics.ts +150 -50
  170. package/src/executors/bounded-exec/lease-wait.ts +1 -0
  171. package/src/executors/bounded-exec/mbx-pool.ts +57 -9
  172. package/src/executors/bounded-exec/mbx-reservations.ts +216 -183
  173. package/src/executors/bounded-exec/parent-grant.ts +3 -0
  174. package/src/executors/bounded-exec/pool-admission.ts +59 -2
  175. package/src/executors/bounded-exec/progress-channel.ts +129 -0
  176. package/src/executors/bounded-exec/progress.ts +39 -3
  177. package/src/executors/bounded-exec/ram-temp.ts +137 -17
  178. package/src/executors/bounded-exec/reservation-connection.ts +221 -0
  179. package/src/executors/bounded-exec/run-cpu-pool.ts +2 -0
  180. package/src/executors/bounded-exec/runner-sizing.ts +13 -8
  181. package/src/executors/bounded-exec/schema.json +1 -1
  182. package/src/executors/bounded-exec/schema.ts +21 -2
  183. package/src/executors/bounded-exec/suite-minimum.ts +81 -0
  184. package/src/executors/bounded-exec/test-result-writer.ts +3 -0
  185. package/src/executors/bounded-exec/test-results.ts +5 -0
  186. package/src/executors/bounded-exec/verdict.ts +83 -22
  187. package/src/executors/cargo-zigbuild/executor.ts +30 -19
  188. package/src/index.ts +357 -166
  189. package/src/managed-files/ci-workflow.ts +13 -17
  190. package/src/managed-files/files.ts +3 -3
  191. package/src/managed-files/generator.ts +21 -6
  192. package/src/managed-files/render-memo.ts +135 -0
  193. package/src/test-program.ts +36 -41
  194. package/src/tool-bins.ts +35 -6
  195. package/src/typecheck-test-policy.ts +49 -24
  196. package/src/workspace-config-policy.ts +161 -43
  197. package/src/workspace-index.ts +40 -5
  198. package/test-timings.json +157 -141
  199. package/managed/raw/patches/ttsc@0.36.3.patch +0 -1660
@@ -14,10 +14,22 @@ import {
14
14
  signalProcessGroup,
15
15
  } from '../../process-group.js';
16
16
  import { AdmissionObservation, type QueueHost } from './admission-queue.js';
17
+ import { atClockDeadline, openAwakeClock } from './awake-clock.js';
17
18
  import { type BoundEvent, type BoundKind, type Bounds, earliestBound, stepBounds, UNARMED } from './bounds.js';
19
+ import {
20
+ type CaseGateServer,
21
+ type GatedCases,
22
+ gateCases,
23
+ runWallMs,
24
+ serveCaseGate,
25
+ TEST_GATE_ENV,
26
+ TEST_GATE_SERVED_ENV,
27
+ } from './case-gate.js';
18
28
  import { DIAGNOSTIC_CONTEXT_ENV, DIAGNOSTIC_MODULE_ENV, grantMemoryEnvironment } from './child-exit-diagnostics.js';
19
- import { type CpuPool, createCpuPool } from './cpu-pool.js';
29
+ import { type CpuPool, createCpuPool, lendParentToken } from './cpu-pool.js';
30
+ import type { CpuRise } from './cpu-rise.js';
20
31
  import { GATEWAY_SOCKET } from './gateway-lease.js';
32
+ import { declaredGoTests, goRunSummaryText } from './go-run-summary.js';
21
33
  import type { HangDiagnostics } from './hang-diagnostics.js';
22
34
  import { captureHangDiagnostics, HANG_CAPTURE_MS } from './hang-diagnostics.js';
23
35
  import { type LeafPeak, PEAK_FD, peakWaiter, readWaiterReport, underWaiter } from './leaf-peak.js';
@@ -30,6 +42,7 @@ import {
30
42
  mbxPool,
31
43
  withCountedReservation,
32
44
  withMbxLeaf,
45
+ withReservationAfterFixture,
33
46
  } from './mbx-pool.js';
34
47
  import { describeUnserved, RESERVATIONS_ENV, type Reservation, reserveCores, runEndpoint } from './mbx-reservations.js';
35
48
  import { killedFromOutside, type RunOutcome, runUntilNotKilledForMemory } from './memory-retry.js';
@@ -47,6 +60,7 @@ import { askParentPool, describeGrant } from './parent-grant.js';
47
60
  import { type LastPeak, readLastPeak } from './peak-charge.js';
48
61
  import { CPU_POOL_ENV, type InheritedCpuPool, readInheritedCpuPool } from './pool-environment.js';
49
62
  import { createRunProgress, withProgressReport, withTestResultReport } from './progress.js';
63
+ import { PROGRESS_SOCKET_ENV, type ProgressChannel, serveProgressChannel } from './progress-channel.js';
50
64
  import {
51
65
  createHostCommands,
52
66
  describeCoverageGap,
@@ -62,7 +76,8 @@ import {
62
76
  import { createRunnerLifecycle, withFixtureLifecycle, withRunnerLifecycle } from './runner-lifecycle.js';
63
77
  import { CPU_TOKENS_ENV, runnerDemand, sizeRunner, sizeRunnerToGrant } from './runner-sizing.js';
64
78
  import { checkRuntimePin, describeRuntimePinRefusal } from './runtime-pin.js';
65
- import type { BoundedExecOptions } from './schema.js';
79
+ import { type BoundedExecOptions, boundedCommand } from './schema.js';
80
+ import { readLastWalls, suiteMinimum } from './suite-minimum.js';
66
81
  import { createTestResultWriter, type TestResultWriter } from './test-result-writer.js';
67
82
  import {
68
83
  type BoundedExecRecord,
@@ -76,6 +91,7 @@ import {
76
91
  judgeOperationalFailure,
77
92
  judgeRun,
78
93
  type NoSpaceSighting,
94
+ type OperationalCause,
79
95
  readJunitReport,
80
96
  taskDirectory,
81
97
  withJunitReport,
@@ -137,6 +153,8 @@ interface RunState {
137
153
  noSpace: Promise<NoSpaceSighting> | null;
138
154
  /** The lock waits the expiry's snapshot named: empty unless that capture completed. Judged with the expiry. */
139
155
  lockWaits: readonly LockWait[];
156
+ /** The processes the expiry's snapshot saw gain CPU time: empty unless that capture completed. Judged with the expiry. */
157
+ cpuRising: readonly CpuRise[];
140
158
  }
141
159
 
142
160
  export interface ProcessTreeKiller {
@@ -305,10 +323,14 @@ export async function runBoundedExec(
305
323
  // pool, read once before a run's clear removes them: a command run again after a kill is charged by the record the
306
324
  // killed run replaced, not by the peak it reached before it was killed.
307
325
  const lastPeak = directory !== null && inherited !== null ? await readLastPeak(directory) : null;
326
+ // The walls its last run recorded size the minimum its reservation names (suite-minimum.ts), read before the same
327
+ // clear removes them.
328
+ const lastWalls = directory !== null ? await readLastWalls(directory) : new Map<string, number>();
308
329
  return runUntilNotKilledForMemory(
309
330
  (alone) =>
310
331
  runCommand(options, context, killer, tempVolume, inherited, captureDiagnostics, owner, queueHost, {
311
332
  lastPeak,
333
+ lastWalls,
312
334
  alone,
313
335
  }),
314
336
  inherited !== null,
@@ -316,9 +338,13 @@ export async function runBoundedExec(
316
338
  );
317
339
  }
318
340
 
319
- /** What a run of the command is told that the run before it was not: the task's charge, and whether it runs alone. */
341
+ /**
342
+ * What a run of the command is told that the run before it was not: the task's charge, the walls its tests last took,
343
+ * and whether it runs alone.
344
+ */
320
345
  interface Attempt {
321
346
  readonly lastPeak: LastPeak | null;
347
+ readonly lastWalls: ReadonlyMap<string, number>;
322
348
  readonly alone: boolean;
323
349
  }
324
350
 
@@ -336,12 +362,12 @@ async function runCommand(
336
362
  ): Promise<BoundedExecResult> {
337
363
  const cwd = resolveCwd(options.cwd, context.root);
338
364
  const directory = context.record?.directory ?? null;
339
- const { lastPeak } = attempt;
365
+ const { lastPeak, lastWalls } = attempt;
340
366
  // A previous run's verdict and report go first, so neither can be read as this run's.
341
367
  if (directory !== null) {
342
368
  await clearTaskDirectory(directory);
343
369
  }
344
- const requested = buildCommand(options);
370
+ const requested = boundedCommand(options);
345
371
  const timeoutMs = options.timeoutMs;
346
372
  const idleTimeoutMs = options.idleTimeoutMs;
347
373
  const killAfterMs = options.killAfterMs ?? DEFAULT_KILL_AFTER_MS;
@@ -361,6 +387,7 @@ async function runCommand(
361
387
  retireError: null,
362
388
  noSpace: null,
363
389
  lockWaits: [],
390
+ cpuRising: [],
364
391
  };
365
392
  const appendStdout = (chunk: Buffer | string): void => {
366
393
  const text = chunk.toString();
@@ -393,13 +420,21 @@ async function runCommand(
393
420
  await writeRecord(recordDirectory, {
394
421
  task: task.id,
395
422
  hash: task.hash,
396
- verdict: judgeOperationalFailure(1, null),
423
+ verdict: judgeOperationalFailure(1, null, { kind: 'executor', message }),
397
424
  peakBytes: null,
398
425
  });
399
426
  }
400
427
  return { success: false, terminalOutput: outputChunks.join('') };
401
428
  };
402
429
 
430
+ let awakeNow = (): number => performance.now();
431
+ if (idleTimeoutMs !== undefined) {
432
+ try {
433
+ awakeNow = await openAwakeClock();
434
+ } catch (error) {
435
+ return refuseUnstarted(`bounded-exec: cannot read its awake idle clock: ${String(error)}\n`);
436
+ }
437
+ }
403
438
  // The fixture pair names one segment: a fixture with no build budget, or a budget with no fixture, is a
404
439
  // target that does not say what it runs, so it does not run. Units are counted at the end of that segment.
405
440
  if (
@@ -582,6 +617,7 @@ async function runCommand(
582
617
  parallelism: chargeParallelism,
583
618
  alone: attempt.alone,
584
619
  task: context.record?.task.id,
620
+ gated: demand.kind === 'nextest' && process.env[TEST_GATE_SERVED_ENV] === '1',
585
621
  });
586
622
  for (const said of parent.said) appendStderr(said);
587
623
  const { grant, refusal: parentRefusal, wait: parentWait } = parent;
@@ -591,15 +627,26 @@ async function runCommand(
591
627
  // With mbx the leaf's admission is mbx's one pool, not a parent's tokens. A leaf mbx does not admit itself asks for
592
628
  // its runner's count, a maximum, and its runner runs on the cores it is granted, which only the command learns
593
629
  // (specs/cowshed/05_gateway.md, "Parallelism inside a job"). No grant and no sub-pool beside it.
594
- const leaf = mbx === null ? null : mbxLeaf(mbx, demand, requested);
630
+ const leaf = mbx === null ? null : mbxLeaf(mbx, demand, requested, process.env[TEST_GATE_SERVED_ENV] === '1');
631
+ // A nextest leaf granted fewer of its parent pool's tokens than it asked takes the case gate where the run's mbx
632
+ // gates cases: its runner starts every thread of its ask, and a case that finds no token of the grant free asks the
633
+ // pool for one of its own (specs/cowshed/05_gateway.md, "A case starts on a core", Inside a hold).
634
+ const lender =
635
+ grant !== null &&
636
+ inherited !== null &&
637
+ demand.kind === 'nextest' &&
638
+ grant.tokens < demand.want &&
639
+ process.env[TEST_GATE_SERVED_ENV] === '1'
640
+ ? { own: grant.tokens, pool: inherited }
641
+ : null;
595
642
  if (grant !== null) {
596
- sized = sizeRunner(demand, requested, grant.tokens);
643
+ sized = sizeRunner(demand, requested, grant.tokens, lender !== null);
597
644
  cpu = { tokens: grant.tokens, want: demand.want, waitedMs };
598
645
  for (const said of describeGrant(grant, { waitedMs, want: demand.want, runner: demand.kind, charge })) {
599
646
  appendStderr(said);
600
647
  }
601
648
  } else if (leaf?.kind === 'reserve') {
602
- sized = { command: sizeRunnerToGrant(leaf.demand, requested), env: {} };
649
+ sized = { command: sizeRunnerToGrant(leaf.demand, requested, leaf.gated), env: {} };
603
650
  } else if (leaf !== null) {
604
651
  sized = sizeRunner(demand, requested, demand.want);
605
652
  }
@@ -609,6 +656,37 @@ async function runCommand(
609
656
  let pool: CpuPool | null = null;
610
657
  let reservation: Reservation | null = null;
611
658
  const releaseReservation = (): void => reservation?.release();
659
+ // A gated nextest leaf's case gate (case-gate.ts; specs/cowshed/05_gateway.md, "A case starts on a core"): served
660
+ // before the command starts, decided from the leaf's grant on, and closed once the command is gone, when every core a
661
+ // case still holds goes back to the pool and the leaf says what its cases ran on.
662
+ let caseGate: CaseGateServer | null = null;
663
+ let gated: GatedCases | null = null;
664
+ let caseGateClosed: Promise<void> | null = null;
665
+ const closeCaseGate = (): Promise<void> => {
666
+ caseGateClosed ??= (async () => {
667
+ const left = (await caseGate?.close()) ?? null;
668
+ if (left !== null) appendStderr(`bounded-exec: ${left}\n`);
669
+ const decided = gated;
670
+ if (decided === null) return;
671
+ decided.releaseAll();
672
+ const { own, lent, lentCoreMs, waitedCaseMs } = decided.cases.summary();
673
+ tell(
674
+ `cowshed: ${grant !== null ? 'cpu-pool' : 'mbx reserve'} cases own=${own} lent=${lent} lent_core_s=${Math.round(lentCoreMs / 1000)} waited_case_s=${Math.round(waitedCaseMs / 1000)}\n`,
675
+ );
676
+ })();
677
+ return caseGateClosed;
678
+ };
679
+ // The run's progress channel (progress-channel.ts), served when the run has an idle bound for its records to restart,
680
+ // and closed once its command is gone. A channel that leaves its socket behind says so; the run's verdict stands.
681
+ let progressChannel: ProgressChannel | null = null;
682
+ let progressClosed: Promise<void> | null = null;
683
+ const closeProgress = (): Promise<void> => {
684
+ progressClosed ??= (async () => {
685
+ const left = (await progressChannel?.close()) ?? null;
686
+ if (left !== null) appendStderr(`bounded-exec: ${left}\n`);
687
+ })();
688
+ return progressClosed;
689
+ };
612
690
  let cpuReleased: Promise<void> | null = null;
613
691
  const releaseCpu = (): Promise<void> => {
614
692
  cpuReleased ??= (async () => {
@@ -631,6 +709,7 @@ async function runCommand(
631
709
  tell,
632
710
  step: (event) => stepTo(event),
633
711
  testing: () => isTesting(),
712
+ awakeNow,
634
713
  queueHost,
635
714
  });
636
715
  // Records the run's verdict when it has a task to record as. An `expiry` is set only when a bound ended the run.
@@ -639,9 +718,10 @@ async function runCommand(
639
718
  exitCode: number,
640
719
  expiry: ExpiredBound | null,
641
720
  elapsedMs: number,
642
- operational: boolean,
721
+ operational: OperationalCause | null,
643
722
  noSpace: NoSpaceSighting | null,
644
723
  lockWaits: readonly LockWait[],
724
+ cpuRising: readonly CpuRise[],
645
725
  peakBytes: number | null,
646
726
  ): Promise<void> => {
647
727
  if (context.record === undefined) {
@@ -667,9 +747,11 @@ async function runCommand(
667
747
  }
668
748
  });
669
749
  const admissionQueue = queue.queueWait(reservation !== null);
750
+ const cause: OperationalCause | null =
751
+ operational ?? (resultFailure === null ? null : { kind: 'executor', message: resultFailure });
670
752
  const verdict =
671
- operational || resultFailure !== null
672
- ? judgeOperationalFailure(exitCode, report)
753
+ cause !== null
754
+ ? judgeOperationalFailure(exitCode, report, cause)
673
755
  : judgeRun({
674
756
  exitCode,
675
757
  elapsedMs,
@@ -677,6 +759,7 @@ async function runCommand(
677
759
  report,
678
760
  noSpace,
679
761
  lockWaits,
762
+ cpuRising,
680
763
  admissionQueue,
681
764
  });
682
765
  if (verdict.outcome === 'bound' && verdict.bound === 'test') {
@@ -695,9 +778,11 @@ async function runCommand(
695
778
  const refuse = async (message: string): Promise<BoundedExecResult> => {
696
779
  appendStderr(message);
697
780
  await releaseCpu();
781
+ await closeProgress();
782
+ await closeCaseGate();
698
783
  // The command never started, so its lease holds nothing of its own to keep.
699
784
  await releaseLease(false);
700
- await recordRun(1, null, Date.now() - asked, false, null, [], null);
785
+ await recordRun(1, null, Date.now() - asked, { kind: 'executor', message: message.trim() }, null, [], [], null);
701
786
  return { success: false, terminalOutput: outputChunks.join('') };
702
787
  };
703
788
  if (parentRefusal !== null) {
@@ -723,6 +808,58 @@ async function runCommand(
723
808
  }
724
809
  pool = created.pool;
725
810
  }
811
+ // A running test reports its own progress on the run's progress channel, and each record restarts the idle bound as a
812
+ // finished test does; a record that is not one fails the run as a broken reporter. A run with no idle bound has
813
+ // nothing for a record to restart and is served none. Only the command learns the socket, from its environment, so
814
+ // these handlers run after it is spawned and its bounds are wired below.
815
+ if (idleTimeoutMs !== undefined) {
816
+ const served = await serveProgressChannel({
817
+ moved: () => {
818
+ if (fixtureDone) stepTo({ kind: 'reported', at: awakeNow(), testing: isTesting() });
819
+ },
820
+ invalid: (detail) => {
821
+ if (lifecycleFailure !== null || state.expiry !== null || state.settled) return;
822
+ lifecycleFailure = { kind: 'executor', message: `progress channel: ${detail}` };
823
+ appendStderr(`bounded-exec: ${lifecycleFailure.message}\n`);
824
+ termination = terminate();
825
+ },
826
+ });
827
+ if (!served.ok) {
828
+ return refuse(
829
+ `bounded-exec: the run's progress channel could not be served (${served.error}); refusing to run the command\n`,
830
+ );
831
+ }
832
+ progressChannel = served.channel;
833
+ }
834
+ // A case that arrives before the leaf's grant has no core to take and no leaf to wait for: its connection is refused,
835
+ // and its shim fails it by name.
836
+ if (reserved?.leaf.gated === true || lender !== null) {
837
+ const served = await serveCaseGate({
838
+ arrived: (id) => gated?.cases.arrived(id) ?? false,
839
+ ended: (id) => gated?.cases.ended(id),
840
+ invalid: (detail) => {
841
+ if (lifecycleFailure !== null || state.expiry !== null || state.settled) return;
842
+ lifecycleFailure = { kind: 'executor', message: `case gate: ${detail}` };
843
+ appendStderr(`bounded-exec: ${lifecycleFailure.message}\n`);
844
+ termination = terminate();
845
+ },
846
+ });
847
+ if (!served.ok) {
848
+ return refuse(
849
+ `bounded-exec: the leaf's case gate could not be served (${served.error}); refusing to run the command\n`,
850
+ );
851
+ }
852
+ caseGate = served.gate;
853
+ if (lender !== null) {
854
+ const gate = served.gate;
855
+ gated = gateCases(
856
+ lender.own,
857
+ lastWalls,
858
+ (id) => gate.start(id),
859
+ (id, left) => lendParentToken(lender.pool, `nextest:${id}`, left),
860
+ );
861
+ }
862
+ }
726
863
  const native = directory === null ? { command: sized.command, env: {} } : withTestResultReport(sized.command);
727
864
  const command =
728
865
  directory === null
@@ -740,11 +877,13 @@ async function runCommand(
740
877
  ...(pool?.environment ?? {}),
741
878
  ...(await grantMemoryEnvironment(grant?.charged ?? null, process.env)),
742
879
  ...(leaf?.kind === 'cargo' ? { MBX_GRANT_FD: String(MBX_GRANT_FD) } : {}),
880
+ ...(progressChannel === null ? {} : { [PROGRESS_SOCKET_ENV]: progressChannel.socket }),
881
+ ...(caseGate === null ? {} : { [TEST_GATE_ENV]: caseGate.socket }),
743
882
  };
744
883
  // A fixture rides as segment 0 of the lifecycle. Without the channel (Windows) it is plain shell text ahead
745
- // of the command: the progress bound runs during it and its own ceiling does not. Under mbx a leaf that names its
746
- // units counts them at the end of that segment, under the fixture's ceiling, its fixture's own output on stderr, and
747
- // reserves after it (`withCountedReservation`).
884
+ // of the command: the progress bound runs during it and its own ceiling does not. Under mbx a reserved leaf's
885
+ // fixture runs before its reservation, its ceiling from spawn (`withReservationAfterFixture`); a leaf that names its
886
+ // units counts them at the end of that segment, its fixture's own output on stderr (`withCountedReservation`).
748
887
  const counted =
749
888
  process.platform !== 'win32' && leaf?.kind === 'reserve' && fixture !== null && options.unitsCommand !== undefined;
750
889
  const segment =
@@ -753,15 +892,29 @@ async function runCommand(
753
892
  process.platform === 'win32' || segment === null ? null : withFixtureLifecycle(segment, command);
754
893
  const hooked = process.platform === 'win32' ? null : (fixtureHooked ?? withRunnerLifecycle(command));
755
894
  const lifecycle = hooked === null ? null : createRunnerLifecycle(hooked.runners);
756
- let lifecycleFailure: string | null = null;
895
+ let lifecycleFailure: OperationalCause | null = null;
896
+ /** Why the executor ended or failed the run whatever its command did, if it did: the verdict's operational cause. */
897
+ /** The cause the executor ended the command for, if it ended it (closures set `lifecycleFailure`). */
898
+ const endedFor = (): OperationalCause | null => lifecycleFailure;
899
+ const operationalCause = (): OperationalCause | null =>
900
+ endedFor() ??
901
+ (state.poolCloseError !== null
902
+ ? { kind: 'executor', message: `cpu pool did not close cleanly: ${state.poolCloseError}` }
903
+ : state.retireError !== null
904
+ ? { kind: 'executor', message: `could not retire the command's processes: ${state.retireError}` }
905
+ : null);
757
906
  const isTesting = (): boolean => lifecycleFailure !== null || (lifecycle?.isTesting() ?? true);
758
907
  const startedAt = Date.now();
759
908
  // With a supervisor, `cowshed subtree-root` asks it for a handle on itself, writes the answer to the handle
760
909
  // descriptor, and execs `/bin/sh -c command` in the same process: the command's pid, group and life are unchanged.
761
910
  const unadmitted = hooked?.command ?? (segment === null ? command : `${segment} && ${command}`);
762
911
  const shellCommand =
763
- leaf?.kind === 'reserve' && counted && fixtureHooked !== null
764
- ? withCountedReservation(fixtureHooked.fixture, fixtureHooked.main, leaf)
912
+ leaf?.kind === 'reserve' && fixtureHooked !== null
913
+ ? (counted ? withCountedReservation : withReservationAfterFixture)(
914
+ fixtureHooked.fixture,
915
+ fixtureHooked.main,
916
+ leaf,
917
+ )
765
918
  : withMbxLeaf(unadmitted, leaf);
766
919
  // Only leaves known to be scheduled can exclude zero-hold time. Others keep lifecycle hang protection.
767
920
  const queued = mbxIdleGated(leaf, sized.command, { ...process.env, ...env });
@@ -845,16 +998,23 @@ async function runCommand(
845
998
  });
846
999
  }
847
1000
 
848
- // Progress is a finished test, not output: a runner's SLOW warnings and a hung test's heartbeat keep printing.
1001
+ // Progress in the output is a finished test, never output itself: a runner's SLOW warnings and a hung test's heartbeat
1002
+ // keep printing. A running test reports its own progress on the progress channel instead.
849
1003
  const progress = createRunProgress(
850
1004
  command,
851
1005
  resultWriter === null
852
1006
  ? undefined
853
- : (result) => {
1007
+ : (closed) => {
1008
+ // nextest times a case from its shim's start: the wait at the gate is taken off its wall and named beside it.
1009
+ const waitMs = gated?.cases.waitedMs(closed.id) ?? 0;
1010
+ const result =
1011
+ closed.durationMs === null || waitMs === 0
1012
+ ? closed
1013
+ : { ...closed, durationMs: runWallMs(closed.durationMs, waitMs), waitMs };
854
1014
  const error = resultWriter?.append(result) ?? null;
855
1015
  if (error !== null && resultFailure === null) {
856
1016
  resultFailure = error;
857
- lifecycleFailure ??= error;
1017
+ lifecycleFailure ??= { kind: 'executor', message: error };
858
1018
  appendStderr(`bounded-exec: ${error}\n`);
859
1019
  termination = terminate();
860
1020
  }
@@ -874,56 +1034,57 @@ async function runCommand(
874
1034
  let fixtureDone = fixture === null || lifecycle === null;
875
1035
  // Whether a segment past the fixture ever announced: without one, the shell's exit is the fixture's.
876
1036
  let mainStarted = fixture === null || lifecycle === null;
877
- let captureTimer: NodeJS.Timeout | undefined;
878
- let idleCapture: { expiresAtMs: number; controller: AbortController; result: IdleDiagnosticResult } | undefined;
1037
+ let cancelCaptureTimer: (() => void) | undefined;
1038
+ let idleCapture:
1039
+ | { scheduledExpiryAtMs: number; controller: AbortController; result: IdleDiagnosticResult }
1040
+ | undefined;
879
1041
  const cancelCapture = (): void => {
880
- clearTimeout(captureTimer);
1042
+ cancelCaptureTimer?.();
881
1043
  idleCapture?.controller.abort();
882
1044
  };
883
- const scheduleCapture = (expiresAtMs: number): void => {
1045
+ const scheduleCapture = (awakeExpiresAtMs: number): void => {
884
1046
  const capture: NonNullable<typeof idleCapture> = {
885
- expiresAtMs,
1047
+ scheduledExpiryAtMs: Date.now() + Math.ceil(Math.max(0, awakeExpiresAtMs - awakeNow())),
886
1048
  controller: new AbortController(),
887
1049
  result: { kind: 'pending', startedAtMs: null },
888
1050
  };
889
1051
  idleCapture = capture;
890
- captureTimer = setTimeout(
891
- () => {
892
- if (state.settled || state.expiry !== null || idleCapture !== capture) return;
893
- const group = child.pid;
894
- if (group === undefined) {
895
- idleCapture.result = { kind: 'failed', detail: 'runner PID unavailable; no holder conclusion' };
896
- return;
897
- }
898
- idleCapture.result = { kind: 'pending', startedAtMs: Date.now() };
899
- void runnerPid
900
- .then((pid) => {
901
- if (pid === null)
902
- throw new Error('runner PID unavailable: its waiter named no command; no holder conclusion');
903
- return captureDiagnostics({
904
- pid,
905
- pgid: process.platform === 'win32' ? null : group,
906
- expiresAtMs: capture.expiresAtMs,
907
- deadlineMs: Math.max(0, Math.min(HANG_CAPTURE_MS, capture.expiresAtMs - Date.now())),
908
- signal: capture.controller.signal,
909
- });
910
- })
911
- .then(
912
- (snapshot) => {
913
- if (idleCapture === capture) idleCapture.result = { kind: 'captured', snapshot };
914
- },
915
- (error: unknown) => {
916
- if (idleCapture === capture) {
917
- idleCapture.result = {
918
- kind: 'failed',
919
- detail: error instanceof Error ? error.message : String(error),
920
- };
921
- }
922
- },
923
- );
924
- },
925
- Math.max(0, expiresAtMs - Date.now() - HANG_CAPTURE_MS),
926
- );
1052
+ cancelCaptureTimer = atClockDeadline(awakeExpiresAtMs - HANG_CAPTURE_MS, awakeNow, () => {
1053
+ if (state.settled || state.expiry !== null || idleCapture !== capture) return;
1054
+ const group = child.pid;
1055
+ if (group === undefined) {
1056
+ idleCapture.result = { kind: 'failed', detail: 'runner PID unavailable; no holder conclusion' };
1057
+ return;
1058
+ }
1059
+ idleCapture.result = { kind: 'pending', startedAtMs: Date.now() };
1060
+ capture.scheduledExpiryAtMs = Date.now() + Math.ceil(Math.max(0, awakeExpiresAtMs - awakeNow()));
1061
+ void runnerPid
1062
+ .then((pid) => {
1063
+ if (capture.controller.signal.aborted || awakeNow() >= awakeExpiresAtMs) return;
1064
+ if (pid === null)
1065
+ throw new Error('runner PID unavailable: its waiter named no command; no holder conclusion');
1066
+ return captureDiagnostics({
1067
+ pid,
1068
+ pgid: process.platform === 'win32' ? null : group,
1069
+ awakeExpiresAtMs,
1070
+ awakeNow,
1071
+ signal: capture.controller.signal,
1072
+ });
1073
+ })
1074
+ .then(
1075
+ (snapshot) => {
1076
+ if (snapshot !== undefined && idleCapture === capture) idleCapture.result = { kind: 'captured', snapshot };
1077
+ },
1078
+ (error: unknown) => {
1079
+ if (idleCapture === capture) {
1080
+ idleCapture.result = {
1081
+ kind: 'failed',
1082
+ detail: error instanceof Error ? error.message : String(error),
1083
+ };
1084
+ }
1085
+ },
1086
+ );
1087
+ });
927
1088
  };
928
1089
  const expiredAs = (kind: BoundKind): ExpiredBound =>
929
1090
  kind === 'idle'
@@ -932,25 +1093,27 @@ async function runCommand(
932
1093
  ? { kind, limitMs: fixture?.timeoutMs ?? 0 }
933
1094
  : { kind, limitMs: timeoutMs ?? 0 };
934
1095
  let checkingIdle = false;
1096
+ const nextBound = () => earliestBound(checkingIdle ? { ...bounds, idle: null } : bounds, awakeNow(), Date.now());
935
1097
  const armBoundTimer = (): void => {
936
1098
  clearTimeout(boundTimer);
937
1099
  if (state.settled || state.expiry !== null) return;
938
- const next = earliestBound(checkingIdle ? { ...bounds, idle: null } : bounds);
1100
+ const next = nextBound();
939
1101
  if (next !== null) {
940
- boundTimer = setTimeout(
941
- () => {
942
- void confirmExpiry(next.kind, next.at);
943
- },
944
- Math.max(0, next.at - Date.now()),
945
- );
1102
+ if (next.kind === 'idle' && idleCapture !== undefined && next.remainingMs > 0) {
1103
+ idleCapture.scheduledExpiryAtMs = Date.now() + Math.ceil(next.remainingMs);
1104
+ }
1105
+ boundTimer = setTimeout(() => void confirmExpiry(), Math.ceil(Math.max(0, next.remainingMs)));
946
1106
  }
947
1107
  };
948
- const confirmExpiry = async (kind: BoundKind, at: number): Promise<void> => {
1108
+ const confirmExpiry = async (): Promise<void> => {
949
1109
  if (state.settled || state.expiry !== null) return;
950
- const scheduled = earliestBound(checkingIdle ? { ...bounds, idle: null } : bounds);
951
- if (scheduled?.kind !== kind || scheduled.at !== at) return;
952
- if (kind === 'idle' && leaf?.kind === 'cargo') {
953
- if (checkingIdle) return;
1110
+ const scheduled = nextBound();
1111
+ if (scheduled === null) return;
1112
+ if (scheduled.remainingMs > 0) {
1113
+ armBoundTimer();
1114
+ return;
1115
+ }
1116
+ if (scheduled.kind === 'idle' && leaf?.kind === 'cargo') {
954
1117
  checkingIdle = true;
955
1118
  armBoundTimer();
956
1119
  try {
@@ -962,14 +1125,14 @@ async function runCommand(
962
1125
  }
963
1126
  }
964
1127
  if (state.settled || state.expiry !== null) return;
965
- const next = earliestBound(bounds);
966
- if (next !== null && next.at <= Date.now()) expire(expiredAs(next.kind));
1128
+ const next = nextBound();
1129
+ if (next !== null && next.remainingMs <= 0) expire(expiredAs(next.kind));
967
1130
  else armBoundTimer();
968
1131
  };
969
1132
  const stepTo = (event: BoundEvent): void => {
970
1133
  if (state.settled || state.expiry !== null) return;
971
1134
  const idleBefore = bounds.idle;
972
- bounds = stepBounds(
1135
+ const next = stepBounds(
973
1136
  bounds,
974
1137
  {
975
1138
  ...(idleTimeoutMs === undefined ? {} : { idleMs: idleTimeoutMs }),
@@ -978,6 +1141,8 @@ async function runCommand(
978
1141
  },
979
1142
  event,
980
1143
  );
1144
+ if (next === bounds) return;
1145
+ bounds = next;
981
1146
  if (bounds.idle !== idleBefore) {
982
1147
  cancelCapture();
983
1148
  idleCapture = undefined;
@@ -990,14 +1155,14 @@ async function runCommand(
990
1155
  // through appendStderr as well, and re-arming on those would let a teardown
991
1156
  // message extend the very bound that just fired.
992
1157
  const updateIdle = (finished: boolean, wasTesting: boolean): void => {
993
- stepTo({ kind: 'progress', at: Date.now(), testing: isTesting(), wasTesting, finished });
1158
+ stepTo({ kind: 'progress', at: awakeNow(), testing: isTesting(), wasTesting, finished });
994
1159
  };
995
1160
  const scanProgress = (stream: 'stdout' | 'stderr', text: Buffer | string): void => {
996
1161
  if (!fixtureDone) return;
997
1162
  updateIdle(progress.scan(stream, text), isTesting());
998
1163
  const error = progress.error();
999
1164
  if (error !== null && lifecycleFailure === null) {
1000
- lifecycleFailure = error;
1165
+ lifecycleFailure = { kind: 'executor', message: error };
1001
1166
  appendStderr(`bounded-exec: ${error}\n`);
1002
1167
  termination = terminate();
1003
1168
  }
@@ -1007,7 +1172,7 @@ async function runCommand(
1007
1172
  if (!(channel instanceof Duplex)) throw new Error('runner lifecycle requires a duplex child descriptor');
1008
1173
  const failLifecycle = (detail: string, wasTesting = isTesting()): void => {
1009
1174
  if (lifecycleFailure !== null || state.expiry !== null) return;
1010
- lifecycleFailure = detail;
1175
+ lifecycleFailure = { kind: 'executor', message: `runner lifecycle failed: ${detail}` };
1011
1176
  appendStderr(`bounded-exec: runner lifecycle failed: ${detail}\n`);
1012
1177
  updateIdle(false, wasTesting);
1013
1178
  };
@@ -1082,6 +1247,8 @@ async function runCommand(
1082
1247
  queue.stop();
1083
1248
  return killChildTree();
1084
1249
  })
1250
+ .finally(closeProgress)
1251
+ .finally(closeCaseGate)
1085
1252
  .finally(releaseCpu)
1086
1253
  .finally(() => releaseLease(true))
1087
1254
  .finally(releaseReservation)
@@ -1090,9 +1257,10 @@ async function runCommand(
1090
1257
  signalToExitCode(signal),
1091
1258
  state.expiry,
1092
1259
  Date.now() - startedAt,
1093
- state.poolCloseError !== null || lifecycleFailure !== null || state.retireError !== null,
1260
+ operationalCause(),
1094
1261
  state.noSpace === null ? null : await state.noSpace,
1095
1262
  state.lockWaits,
1263
+ state.cpuRising,
1096
1264
  null,
1097
1265
  ),
1098
1266
  )
@@ -1145,7 +1313,7 @@ async function runCommand(
1145
1313
  const elapsedMs = Date.now() - startedAt;
1146
1314
  appendStderr(
1147
1315
  bound.kind === 'idle'
1148
- ? `\nCommand made no progress: no test finished for ${bound.limitMs}ms (idleTimeoutMs=${bound.limitMs}) after ${elapsedMs}ms of runtime; ${
1316
+ ? `\nCommand made no progress: no test finished or reported progress for ${bound.limitMs}ms (idleTimeoutMs=${bound.limitMs}) after ${elapsedMs}ms of runtime; ${
1149
1317
  bound.tests.length > 0
1150
1318
  ? `still running: ${bound.tests.join(', ')}`
1151
1319
  : 'the runner named no test still running'
@@ -1161,10 +1329,11 @@ async function runCommand(
1161
1329
  if (bound.kind === 'idle') {
1162
1330
  const observed = idleCapture?.result;
1163
1331
  state.lockWaits = observed?.kind === 'captured' ? observed.snapshot.lockWaits : [];
1332
+ state.cpuRising = observed?.kind === 'captured' ? observed.snapshot.cpuRising : [];
1164
1333
  appendStderr(
1165
1334
  `Hang diagnostics: runner pid=${runner ?? 'unavailable'} pgid=${
1166
1335
  process.platform === 'win32' ? 'unavailable' : (child.pid ?? 'unavailable')
1167
- } expiryAtMs=${Date.now()} scheduledExpiryAtMs=${idleCapture?.expiresAtMs ?? 'unavailable'}\n`,
1336
+ } expiryAtMs=${Date.now()} scheduledExpiryAtMs=${idleCapture?.scheduledExpiryAtMs ?? 'unavailable'}\n`,
1168
1337
  );
1169
1338
  if (observed?.kind === 'captured') {
1170
1339
  appendStderr(
@@ -1252,22 +1421,27 @@ async function runCommand(
1252
1421
  // fire on a command the fixture already handed over to.
1253
1422
  stepTo({
1254
1423
  kind: 'spawn',
1255
- at: Date.now(),
1424
+ at: awakeNow(),
1425
+ wallAt: Date.now(),
1256
1426
  testing: isTesting(),
1257
1427
  fixture: fixture !== null && lifecycle !== null,
1258
- fixtureFirst: counted && lifecycle !== null,
1428
+ fixtureFirst: leaf?.kind === 'reserve' && fixtureHooked !== null,
1259
1429
  queued,
1260
1430
  });
1261
1431
  if (leaf !== null) {
1262
1432
  const channel = child.stdio.at(MBX_GRANT_FD);
1263
1433
  if (!(channel instanceof Duplex)) throw new Error('an mbx leaf requires its mbx descriptor');
1264
- const failGrant = (detail: string): void => {
1434
+ const descriptorFailure = (detail: string): OperationalCause => ({
1435
+ kind: 'executor',
1436
+ message: `mbx grant descriptor: ${detail}`,
1437
+ });
1438
+ const failGrant = (cause: OperationalCause): void => {
1265
1439
  if (lifecycleFailure !== null || state.expiry !== null || state.settled) return;
1266
- lifecycleFailure ??= `mbx grant descriptor: ${detail}`;
1267
- appendStderr(`bounded-exec: ${lifecycleFailure}\n`);
1440
+ lifecycleFailure = cause;
1441
+ appendStderr(`bounded-exec: ${cause.message}\n`);
1268
1442
  termination = terminate();
1269
1443
  };
1270
- channel.on('error', (error) => failGrant(error.message));
1444
+ channel.on('error', (error) => failGrant(descriptorFailure(error.message)));
1271
1445
  if (reserved !== null) {
1272
1446
  // The shell asks for its cores in one line and reads the count granted (mbx-pool.ts, `reservation`). A refusal
1273
1447
  // ends the descriptor unanswered: the shell's read fails, and nothing runs.
@@ -1276,7 +1450,7 @@ async function runCommand(
1276
1450
  channel.setEncoding('utf8');
1277
1451
  channel.on('data', (text: string) => {
1278
1452
  if (reservation !== null) {
1279
- failGrant(`the command asked for cores a second time: ${JSON.stringify(text)}`);
1453
+ failGrant(descriptorFailure(`the command asked for cores a second time: ${JSON.stringify(text)}`));
1280
1454
  return;
1281
1455
  }
1282
1456
  asked += text;
@@ -1285,17 +1459,30 @@ async function runCommand(
1285
1459
  const line = asked.slice(0, end);
1286
1460
  const cpus = askedCores(line, reservedLeaf.demand.want);
1287
1461
  if (cpus === null) {
1288
- failGrant(`the command asked for ${JSON.stringify(line)} cores, not 1 to ${reservedLeaf.demand.want}`);
1462
+ failGrant(
1463
+ descriptorFailure(
1464
+ `the command asked for ${JSON.stringify(line)} cores, not 1 to ${reservedLeaf.demand.want}`,
1465
+ ),
1466
+ );
1289
1467
  channel.end();
1290
1468
  return;
1291
1469
  }
1292
1470
  queue.waiting();
1293
- tell(`cowshed: mbx reserve wait start cpus=${cpus} priority=${reservedLeaf.priority}\n`);
1294
- const held = reserveCores(endpoint, cpus, reservedLeaf.priority);
1471
+ // A whole-suite test leaf names the fewest threads at which its recorded walls keep a test finishing within
1472
+ // its own no-progress bound (specs/cowshed/05_gateway.md, "A minimum for a whole suite").
1473
+ const min = idleTimeoutMs === undefined ? 1 : suiteMinimum([...lastWalls.values()], cpus, idleTimeoutMs);
1474
+ const named = min > 1 ? ` min=${min}` : '';
1475
+ tell(`cowshed: mbx reserve wait start cpus=${cpus}${named} priority=${reservedLeaf.priority}\n`);
1476
+ const held = reserveCores(endpoint, cpus, reservedLeaf.priority, min);
1295
1477
  reservation = held;
1296
1478
  void held.granted.then((granted) => {
1297
1479
  if (!granted.ok) {
1298
- failGrant(`mbx did not grant the reservation: ${granted.error}`);
1480
+ failGrant({
1481
+ kind: 'reservation-refused',
1482
+ endpoint: granted.refusal.endpoint,
1483
+ errno: granted.refusal.errno,
1484
+ message: `mbx did not grant the reservation: ${granted.refusal.message}`,
1485
+ });
1299
1486
  channel.end();
1300
1487
  return;
1301
1488
  }
@@ -1303,8 +1490,20 @@ async function runCommand(
1303
1490
  if (state.settled || state.expiry !== null) return;
1304
1491
  queue.admitted = true;
1305
1492
  tell(`cowshed: mbx reserve granted elapsed=${Date.now() - startedAt}ms priority=${reservedLeaf.priority}\n`);
1493
+ if (caseGate !== null) {
1494
+ const gate = caseGate;
1495
+ gated = gateCases(
1496
+ granted.cpus,
1497
+ lastWalls,
1498
+ (id) => gate.start(id),
1499
+ (id) => {
1500
+ const held = reserveCores(endpoint, 1, 'low', 1, `nextest:${id}`);
1501
+ return { granted: held.granted.then((answer) => answer.ok), release: () => held.release() };
1502
+ },
1503
+ );
1504
+ }
1306
1505
  channel.end(`${granted.cpus}\n`);
1307
- stepTo({ kind: 'granted', at: Date.now(), testing: isTesting(), fixture: !fixtureDone });
1506
+ stepTo({ kind: 'granted', at: awakeNow(), wallAt: Date.now(), testing: isTesting(), fixture: !fixtureDone });
1308
1507
  void granted.lost.then((why) => {
1309
1508
  if (state.settled || cpuReleased !== null) return;
1310
1509
  appendStderr(
@@ -1324,11 +1523,17 @@ async function runCommand(
1324
1523
  said = true;
1325
1524
  tell(`cowshed: mbx cargo granted elapsed=${Date.now() - startedAt}ms\n`);
1326
1525
  }
1327
- stepTo({ kind: 'granted', at: Date.now(), testing: isTesting(), fixture: !fixtureDone });
1526
+ stepTo({
1527
+ kind: 'granted',
1528
+ at: awakeNow(),
1529
+ wallAt: Date.now(),
1530
+ testing: isTesting(),
1531
+ fixture: !fixtureDone,
1532
+ });
1328
1533
  } else if (byte === 114) {
1329
- stepTo({ kind: 'released', at: Date.now() });
1534
+ stepTo({ kind: 'released', at: awakeNow() });
1330
1535
  } else {
1331
- failGrant(`unexpected byte ${byte}`);
1536
+ failGrant(descriptorFailure(`unexpected byte ${byte}`));
1332
1537
  return;
1333
1538
  }
1334
1539
  }
@@ -1365,18 +1570,25 @@ async function runCommand(
1365
1570
  clearTimeout(boundTimer);
1366
1571
  cancelCapture();
1367
1572
  removeSignalHandlers();
1368
- // The runner is gone. Its sub-pool closes first, so no descendant still draws on tokens the grant returns;
1369
- // then the grant goes back to its parent or the host.
1573
+ // The runner is gone, and with it every test that could report on the run's progress channel. Its sub-pool closes
1574
+ // first, so no descendant still draws on tokens the grant returns; then the grant goes back to its parent or the host.
1575
+ await closeProgress();
1576
+ await closeCaseGate();
1370
1577
  await releaseCpu();
1371
1578
 
1372
1579
  const code = exitCode ?? signalToExitCode(exitSignal);
1373
1580
  if (code !== 0 && state.expiry === null) {
1374
- // No bound of this executor ended it, so a signal it died of came from outside (the kernel's OOM killer sends
1375
- // SIGKILL): the signal is named, never folded into a bare status.
1581
+ // The executor ended it for a cause of its own, which is said; otherwise no bound of this executor ended it, so a
1582
+ // signal it died of came from outside (the kernel's OOM killer sends SIGKILL): the signal is named, never folded
1583
+ // into a bare status.
1584
+ const signalled = exitCode === null && exitSignal !== null;
1585
+ const endedBy = endedFor();
1376
1586
  const ended =
1377
- exitCode === null && exitSignal !== null
1378
- ? `was killed by ${exitSignal} (status ${code})`
1379
- : `exited with status ${code}`;
1587
+ endedBy !== null
1588
+ ? `was ended by bounded-exec because ${endedBy.message} (${signalled ? `${exitSignal}, ` : ''}status ${code})`
1589
+ : signalled
1590
+ ? `was killed by ${exitSignal} (status ${code})`
1591
+ : `exited with status ${code}`;
1380
1592
  appendStderr(
1381
1593
  fixture !== null && !mainStarted
1382
1594
  ? `Fixture command ${ended}: ${segment ?? fixture.command}\n`
@@ -1411,10 +1623,29 @@ async function runCommand(
1411
1623
  }
1412
1624
  const peakBytes = reported?.kind === 'measured' ? reported.bytes : null;
1413
1625
 
1626
+ // A Go run sums nothing itself, and a reader that keeps part of a long output cannot count it: the sum ends the output.
1627
+ const goRun = progress.goSummary();
1628
+ if (goRun !== null) {
1629
+ appendStdout(
1630
+ goRunSummaryText(
1631
+ goRun,
1632
+ declaredGoTests(goRun.failedTests, cwd, context.root, mergeEnv(lease?.directory ?? ordinaryTemp, env)),
1633
+ ),
1634
+ );
1635
+ }
1414
1636
  // Broken resource ownership, a lost native lifecycle, or leftovers that could not be retired fail even if a shell
1415
1637
  // branch later exited 0: its verdict is an ordinary operational failure, never a successful or silently unbounded run.
1416
- const operationalFailure = state.poolCloseError !== null || lifecycleFailure !== null || state.retireError !== null;
1417
- await recordRun(code, state.expiry, Date.now() - startedAt, operationalFailure, seen, state.lockWaits, peakBytes);
1638
+ const operationalFailure = operationalCause();
1639
+ await recordRun(
1640
+ code,
1641
+ state.expiry,
1642
+ Date.now() - startedAt,
1643
+ operationalFailure,
1644
+ seen,
1645
+ state.lockWaits,
1646
+ state.cpuRising,
1647
+ peakBytes,
1648
+ );
1418
1649
  const killedForMemory = killedFromOutside({
1419
1650
  sentSignal: state.sentSignal,
1420
1651
  bounded: state.expiry !== null,
@@ -1422,7 +1653,7 @@ async function runCommand(
1422
1653
  code,
1423
1654
  });
1424
1655
  return {
1425
- success: !operationalFailure && resultFailure === null && state.expiry === null && code === 0,
1656
+ success: operationalFailure === null && resultFailure === null && state.expiry === null && code === 0,
1426
1657
  terminalOutput: outputChunks.join(''),
1427
1658
  ...(killedForMemory ? { killedForMemory: true } : {}),
1428
1659
  };
@@ -1443,19 +1674,6 @@ function resolveCwd(cwd: string | undefined, root: string): string {
1443
1674
  return isAbsolute(cwd) ? cwd : join(root, cwd);
1444
1675
  }
1445
1676
 
1446
- function buildCommand(options: BoundedExecOptions): string {
1447
- const parts = [options.command];
1448
- if (Array.isArray(options.args)) {
1449
- parts.push(...options.args);
1450
- } else if (options.args) {
1451
- parts.push(options.args);
1452
- }
1453
- if (options.forwardAllArgs !== false && options.__unparsed__?.length) {
1454
- parts.push(...options.__unparsed__);
1455
- }
1456
- return parts.join(' ');
1457
- }
1458
-
1459
1677
  /** Owned scratch is the command's TMPDIR unless the target names one itself. */
1460
1678
  function mergeEnv(directory: string | null, env: Record<string, string>): NodeJS.ProcessEnv {
1461
1679
  const inherited = withoutPoolEnv(process.env);