@smoothbricks/nx-plugin 0.7.17 → 0.7.19

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (199) hide show
  1. package/README.md +315 -114
  2. package/dist/.build-inputs.json +47 -38
  3. package/dist/built-output-packages.js +1 -1
  4. package/dist/bun-lock-scope.d.ts +128 -20
  5. package/dist/bun-lock-scope.d.ts.map +1 -1
  6. package/dist/bun-lock-scope.js +267 -44
  7. package/dist/cargo-closure-input.d.ts +8 -6
  8. package/dist/cargo-closure-input.d.ts.map +1 -1
  9. package/dist/cargo-closure-input.js +15 -12
  10. package/dist/cargo-source-hash.d.ts +26 -25
  11. package/dist/cargo-source-hash.d.ts.map +1 -1
  12. package/dist/cargo-source-hash.js +84 -105
  13. package/dist/cargo-toolchain-paths.d.ts +2 -2
  14. package/dist/cargo-toolchain-paths.js +2 -2
  15. package/dist/cargo-workspace.d.ts +20 -2
  16. package/dist/cargo-workspace.d.ts.map +1 -1
  17. package/dist/cargo-workspace.js +43 -22
  18. package/dist/ensure-built-receipt.d.ts +3 -2
  19. package/dist/ensure-built-receipt.d.ts.map +1 -1
  20. package/dist/ensure-built-receipt.js +3 -2
  21. package/dist/eslint-config-closure.d.ts +8 -3
  22. package/dist/eslint-config-closure.d.ts.map +1 -1
  23. package/dist/eslint-config-closure.js +24 -5
  24. package/dist/executors/bounded-exec/admission-queue.d.ts +3 -1
  25. package/dist/executors/bounded-exec/admission-queue.d.ts.map +1 -1
  26. package/dist/executors/bounded-exec/admission-queue.js +9 -10
  27. package/dist/executors/bounded-exec/awake-clock.d.ts +5 -0
  28. package/dist/executors/bounded-exec/awake-clock.d.ts.map +1 -0
  29. package/dist/executors/bounded-exec/awake-clock.js +22 -0
  30. package/dist/executors/bounded-exec/bounds.d.ts +19 -6
  31. package/dist/executors/bounded-exec/bounds.d.ts.map +1 -1
  32. package/dist/executors/bounded-exec/bounds.js +22 -15
  33. package/dist/executors/bounded-exec/case-gate.d.ts +114 -0
  34. package/dist/executors/bounded-exec/case-gate.d.ts.map +1 -0
  35. package/dist/executors/bounded-exec/case-gate.js +342 -0
  36. package/dist/executors/bounded-exec/cpu-pool-thread.d.ts +1 -0
  37. package/dist/executors/bounded-exec/cpu-pool-thread.d.ts.map +1 -1
  38. package/dist/executors/bounded-exec/cpu-pool-thread.js +2 -2
  39. package/dist/executors/bounded-exec/cpu-pool.d.ts +28 -1
  40. package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -1
  41. package/dist/executors/bounded-exec/cpu-pool.js +102 -16
  42. package/dist/executors/bounded-exec/cpu-rise.d.ts +73 -0
  43. package/dist/executors/bounded-exec/cpu-rise.d.ts.map +1 -0
  44. package/dist/executors/bounded-exec/cpu-rise.js +111 -0
  45. package/dist/executors/bounded-exec/endpoint-life.d.ts +146 -0
  46. package/dist/executors/bounded-exec/endpoint-life.d.ts.map +1 -0
  47. package/dist/executors/bounded-exec/endpoint-life.js +228 -0
  48. package/dist/executors/bounded-exec/executor.d.ts +1 -1
  49. package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
  50. package/dist/executors/bounded-exec/executor.js +256 -86
  51. package/dist/executors/bounded-exec/gateway-lease.d.ts +1 -1
  52. package/dist/executors/bounded-exec/gateway-lease.d.ts.map +1 -1
  53. package/dist/executors/bounded-exec/gateway-lease.js +8 -1
  54. package/dist/executors/bounded-exec/go-run-summary.d.ts +51 -0
  55. package/dist/executors/bounded-exec/go-run-summary.d.ts.map +1 -0
  56. package/dist/executors/bounded-exec/go-run-summary.js +98 -0
  57. package/dist/executors/bounded-exec/hang-diagnostics.d.ts +11 -3
  58. package/dist/executors/bounded-exec/hang-diagnostics.d.ts.map +1 -1
  59. package/dist/executors/bounded-exec/hang-diagnostics.js +125 -47
  60. package/dist/executors/bounded-exec/lease-wait.d.ts +1 -0
  61. package/dist/executors/bounded-exec/lease-wait.d.ts.map +1 -1
  62. package/dist/executors/bounded-exec/lease-wait.js +1 -0
  63. package/dist/executors/bounded-exec/mbx-pool.d.ts +42 -7
  64. package/dist/executors/bounded-exec/mbx-pool.d.ts.map +1 -1
  65. package/dist/executors/bounded-exec/mbx-pool.js +45 -8
  66. package/dist/executors/bounded-exec/mbx-reservations.d.ts +12 -6
  67. package/dist/executors/bounded-exec/mbx-reservations.d.ts.map +1 -1
  68. package/dist/executors/bounded-exec/mbx-reservations.js +190 -284
  69. package/dist/executors/bounded-exec/parent-grant.d.ts +2 -0
  70. package/dist/executors/bounded-exec/parent-grant.d.ts.map +1 -1
  71. package/dist/executors/bounded-exec/parent-grant.js +1 -0
  72. package/dist/executors/bounded-exec/pool-admission.d.ts +29 -3
  73. package/dist/executors/bounded-exec/pool-admission.d.ts.map +1 -1
  74. package/dist/executors/bounded-exec/pool-admission.js +47 -1
  75. package/dist/executors/bounded-exec/progress-channel.d.ts +38 -0
  76. package/dist/executors/bounded-exec/progress-channel.d.ts.map +1 -0
  77. package/dist/executors/bounded-exec/progress-channel.js +113 -0
  78. package/dist/executors/bounded-exec/progress.d.ts +5 -1
  79. package/dist/executors/bounded-exec/progress.d.ts.map +1 -1
  80. package/dist/executors/bounded-exec/progress.js +31 -1
  81. package/dist/executors/bounded-exec/ram-temp.d.ts +24 -1
  82. package/dist/executors/bounded-exec/ram-temp.d.ts.map +1 -1
  83. package/dist/executors/bounded-exec/ram-temp.js +123 -16
  84. package/dist/executors/bounded-exec/reservation-connection.d.ts +90 -0
  85. package/dist/executors/bounded-exec/reservation-connection.d.ts.map +1 -0
  86. package/dist/executors/bounded-exec/reservation-connection.js +248 -0
  87. package/dist/executors/bounded-exec/run-cpu-pool.d.ts.map +1 -1
  88. package/dist/executors/bounded-exec/run-cpu-pool.js +3 -0
  89. package/dist/executors/bounded-exec/runner-sizing.d.ts +9 -5
  90. package/dist/executors/bounded-exec/runner-sizing.d.ts.map +1 -1
  91. package/dist/executors/bounded-exec/runner-sizing.js +13 -9
  92. package/dist/executors/bounded-exec/schema.d.ts +7 -2
  93. package/dist/executors/bounded-exec/schema.d.ts.map +1 -1
  94. package/dist/executors/bounded-exec/schema.js +17 -1
  95. package/dist/executors/bounded-exec/suite-minimum.d.ts +21 -0
  96. package/dist/executors/bounded-exec/suite-minimum.d.ts.map +1 -0
  97. package/dist/executors/bounded-exec/suite-minimum.js +85 -0
  98. package/dist/executors/bounded-exec/test-result-writer.d.ts +2 -0
  99. package/dist/executors/bounded-exec/test-result-writer.d.ts.map +1 -1
  100. package/dist/executors/bounded-exec/test-result-writer.js +1 -0
  101. package/dist/executors/bounded-exec/test-results.d.ts +5 -0
  102. package/dist/executors/bounded-exec/test-results.d.ts.map +1 -1
  103. package/dist/executors/bounded-exec/test-results.js +2 -2
  104. package/dist/executors/bounded-exec/verdict.d.ts +39 -10
  105. package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
  106. package/dist/executors/bounded-exec/verdict.js +60 -14
  107. package/dist/executors/cargo-zigbuild/executor.d.ts +10 -8
  108. package/dist/executors/cargo-zigbuild/executor.d.ts.map +1 -1
  109. package/dist/executors/cargo-zigbuild/executor.js +32 -19
  110. package/dist/index.d.ts.map +1 -1
  111. package/dist/index.js +294 -152
  112. package/dist/managed-files/ci-workflow.d.ts +4 -1
  113. package/dist/managed-files/ci-workflow.d.ts.map +1 -1
  114. package/dist/managed-files/ci-workflow.js +9 -14
  115. package/dist/managed-files/files.d.ts +3 -3
  116. package/dist/managed-files/files.js +3 -3
  117. package/dist/managed-files/generator.d.ts +5 -1
  118. package/dist/managed-files/generator.d.ts.map +1 -1
  119. package/dist/managed-files/generator.js +22 -7
  120. package/dist/managed-files/render-memo.d.ts +13 -0
  121. package/dist/managed-files/render-memo.d.ts.map +1 -0
  122. package/dist/managed-files/render-memo.js +94 -0
  123. package/dist/task-results.js +2 -1
  124. package/dist/test-program.d.ts +11 -6
  125. package/dist/test-program.d.ts.map +1 -1
  126. package/dist/test-program.js +39 -35
  127. package/dist/tool-bins.d.ts +12 -0
  128. package/dist/tool-bins.d.ts.map +1 -1
  129. package/dist/tool-bins.js +31 -6
  130. package/dist/typecheck-test-policy.d.ts +10 -1
  131. package/dist/typecheck-test-policy.d.ts.map +1 -1
  132. package/dist/typecheck-test-policy.js +40 -24
  133. package/dist/workspace-config-policy.d.ts.map +1 -1
  134. package/dist/workspace-config-policy.js +135 -42
  135. package/dist/workspace-index.d.ts +14 -1
  136. package/dist/workspace-index.d.ts.map +1 -1
  137. package/dist/workspace-index.js +36 -4
  138. package/executors.json +2 -0
  139. package/managed/raw/envrc +24 -0
  140. package/managed/raw/patches/nx@24.0.3.patch +1851 -221
  141. package/managed/raw/patches/ttsc@0.37.1.patch +1055 -0
  142. package/managed/raw/tooling/direnv/developer-links.ts +122 -14
  143. package/managed/raw/tooling/direnv/devenv.smoo.nix +20 -19
  144. package/managed/templates/github/actions/save-nix-devenv/action.yml +19 -1
  145. package/managed/templates/github/actions/setup-devenv/action.yml +2 -0
  146. package/package.json +68 -11
  147. package/runtime/nx-publish/.build-inputs.json +2 -2
  148. package/runtime/nx-publish/cache_adapter.js +293 -3
  149. package/runtime/nx-publish/nx_publish.js +36 -14
  150. package/src/built-output-packages.ts +1 -1
  151. package/src/bun-lock-scope.ts +293 -40
  152. package/src/cargo-closure-input.ts +20 -12
  153. package/src/cargo-source-hash.ts +119 -145
  154. package/src/cargo-toolchain-paths.ts +2 -2
  155. package/src/cargo-workspace.ts +59 -21
  156. package/src/ensure-built-receipt.ts +3 -2
  157. package/src/eslint-config-closure.ts +27 -6
  158. package/src/executors/bounded-exec/admission-queue.ts +11 -10
  159. package/src/executors/bounded-exec/awake-clock.ts +20 -0
  160. package/src/executors/bounded-exec/bounds.ts +41 -19
  161. package/src/executors/bounded-exec/case-gate.ts +365 -0
  162. package/src/executors/bounded-exec/cpu-pool-thread.ts +3 -3
  163. package/src/executors/bounded-exec/cpu-pool.ts +139 -12
  164. package/src/executors/bounded-exec/cpu-rise.ts +156 -0
  165. package/src/executors/bounded-exec/endpoint-life.ts +279 -0
  166. package/src/executors/bounded-exec/executor.ts +341 -123
  167. package/src/executors/bounded-exec/gateway-lease.ts +6 -0
  168. package/src/executors/bounded-exec/go-run-summary.ts +137 -0
  169. package/src/executors/bounded-exec/hang-diagnostics.ts +150 -50
  170. package/src/executors/bounded-exec/lease-wait.ts +1 -0
  171. package/src/executors/bounded-exec/mbx-pool.ts +57 -9
  172. package/src/executors/bounded-exec/mbx-reservations.ts +216 -183
  173. package/src/executors/bounded-exec/parent-grant.ts +3 -0
  174. package/src/executors/bounded-exec/pool-admission.ts +59 -2
  175. package/src/executors/bounded-exec/progress-channel.ts +129 -0
  176. package/src/executors/bounded-exec/progress.ts +39 -3
  177. package/src/executors/bounded-exec/ram-temp.ts +137 -17
  178. package/src/executors/bounded-exec/reservation-connection.ts +221 -0
  179. package/src/executors/bounded-exec/run-cpu-pool.ts +2 -0
  180. package/src/executors/bounded-exec/runner-sizing.ts +13 -8
  181. package/src/executors/bounded-exec/schema.json +1 -1
  182. package/src/executors/bounded-exec/schema.ts +21 -2
  183. package/src/executors/bounded-exec/suite-minimum.ts +81 -0
  184. package/src/executors/bounded-exec/test-result-writer.ts +3 -0
  185. package/src/executors/bounded-exec/test-results.ts +5 -0
  186. package/src/executors/bounded-exec/verdict.ts +83 -22
  187. package/src/executors/cargo-zigbuild/executor.ts +30 -19
  188. package/src/index.ts +357 -166
  189. package/src/managed-files/ci-workflow.ts +13 -17
  190. package/src/managed-files/files.ts +3 -3
  191. package/src/managed-files/generator.ts +21 -6
  192. package/src/managed-files/render-memo.ts +135 -0
  193. package/src/test-program.ts +36 -41
  194. package/src/tool-bins.ts +35 -6
  195. package/src/typecheck-test-policy.ts +49 -24
  196. package/src/workspace-config-policy.ts +161 -43
  197. package/src/workspace-index.ts +40 -5
  198. package/test-timings.json +157 -141
  199. package/managed/raw/patches/ttsc@0.36.3.patch +0 -1660
@@ -8,13 +8,16 @@ import { Duplex, Readable } from 'node:stream';
8
8
  import { workspaceDataDirectoryForWorkspace } from 'nx/src/utils/cache-directory.js';
9
9
  import { DEADLINE_PATIENCE, describeMembersLeft, killProcessGroup, processGroupHasMembers, signalProcessGroup, } from '../../process-group.js';
10
10
  import { AdmissionObservation } from './admission-queue.js';
11
+ import { atClockDeadline, openAwakeClock } from './awake-clock.js';
11
12
  import { earliestBound, stepBounds, UNARMED } from './bounds.js';
13
+ import { gateCases, runWallMs, serveCaseGate, TEST_GATE_ENV, TEST_GATE_SERVED_ENV, } from './case-gate.js';
12
14
  import { DIAGNOSTIC_CONTEXT_ENV, DIAGNOSTIC_MODULE_ENV, grantMemoryEnvironment } from './child-exit-diagnostics.js';
13
- import { createCpuPool } from './cpu-pool.js';
15
+ import { createCpuPool, lendParentToken } from './cpu-pool.js';
14
16
  import { GATEWAY_SOCKET } from './gateway-lease.js';
17
+ import { declaredGoTests, goRunSummaryText } from './go-run-summary.js';
15
18
  import { captureHangDiagnostics, HANG_CAPTURE_MS } from './hang-diagnostics.js';
16
19
  import { PEAK_FD, peakWaiter, readWaiterReport, underWaiter } from './leaf-peak.js';
17
- import { askedCores, MBX_GRANT_FD, mbxIdleGated, mbxLeaf, mbxPool, withCountedReservation, withMbxLeaf, } from './mbx-pool.js';
20
+ import { askedCores, MBX_GRANT_FD, mbxIdleGated, mbxLeaf, mbxPool, withCountedReservation, withMbxLeaf, withReservationAfterFixture, } from './mbx-pool.js';
18
21
  import { describeUnserved, RESERVATIONS_ENV, reserveCores, runEndpoint } from './mbx-reservations.js';
19
22
  import { killedFromOutside, runUntilNotKilledForMemory } from './memory-retry.js';
20
23
  import { createNoSpaceWatch } from './no-space.js';
@@ -23,10 +26,13 @@ import { askParentPool, describeGrant } from './parent-grant.js';
23
26
  import { readLastPeak } from './peak-charge.js';
24
27
  import { CPU_POOL_ENV, readInheritedCpuPool } from './pool-environment.js';
25
28
  import { createRunProgress, withProgressReport, withTestResultReport } from './progress.js';
29
+ import { PROGRESS_SOCKET_ENV, serveProgressChannel } from './progress-channel.js';
26
30
  import { createHostCommands, describeCoverageGap, describeRamTempError, describeSignalled, LEFTOVER_GRACE_MS, PRODUCTION, RAM_TEMP_CAPACITY_BYTES, RamTempVolume, ramTempPaths, } from './ram-temp.js';
27
31
  import { createRunnerLifecycle, withFixtureLifecycle, withRunnerLifecycle } from './runner-lifecycle.js';
28
32
  import { CPU_TOKENS_ENV, runnerDemand, sizeRunner, sizeRunnerToGrant } from './runner-sizing.js';
29
33
  import { checkRuntimePin, describeRuntimePinRefusal } from './runtime-pin.js';
34
+ import { boundedCommand } from './schema.js';
35
+ import { readLastWalls, suiteMinimum } from './suite-minimum.js';
30
36
  import { createTestResultWriter } from './test-result-writer.js';
31
37
  import { BUN_TEST, clearTaskDirectory, describeNoSpace, JUNIT_FILE, JUNIT_REPORT_ENV, judgeOperationalFailure, judgeRun, readJunitReport, taskDirectory, withJunitReport, writeRecord, } from './verdict.js';
32
38
  const DEFAULT_KILL_AFTER_MS = 10_000;
@@ -159,8 +165,12 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
159
165
  // pool, read once before a run's clear removes them: a command run again after a kill is charged by the record the
160
166
  // killed run replaced, not by the peak it reached before it was killed.
161
167
  const lastPeak = directory !== null && inherited !== null ? await readLastPeak(directory) : null;
168
+ // The walls its last run recorded size the minimum its reservation names (suite-minimum.ts), read before the same
169
+ // clear removes them.
170
+ const lastWalls = directory !== null ? await readLastWalls(directory) : new Map();
162
171
  return runUntilNotKilledForMemory((alone) => runCommand(options, context, killer, tempVolume, inherited, captureDiagnostics, owner, queueHost, {
163
172
  lastPeak,
173
+ lastWalls,
164
174
  alone,
165
175
  }), inherited !== null, context.record?.task.id);
166
176
  }
@@ -168,12 +178,12 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
168
178
  async function runCommand(options, context, killer, tempVolume, inherited, captureDiagnostics, owner, queueHost, attempt) {
169
179
  const cwd = resolveCwd(options.cwd, context.root);
170
180
  const directory = context.record?.directory ?? null;
171
- const { lastPeak } = attempt;
181
+ const { lastPeak, lastWalls } = attempt;
172
182
  // A previous run's verdict and report go first, so neither can be read as this run's.
173
183
  if (directory !== null) {
174
184
  await clearTaskDirectory(directory);
175
185
  }
176
- const requested = buildCommand(options);
186
+ const requested = boundedCommand(options);
177
187
  const timeoutMs = options.timeoutMs;
178
188
  const idleTimeoutMs = options.idleTimeoutMs;
179
189
  const killAfterMs = options.killAfterMs ?? DEFAULT_KILL_AFTER_MS;
@@ -192,6 +202,7 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
192
202
  retireError: null,
193
203
  noSpace: null,
194
204
  lockWaits: [],
205
+ cpuRising: [],
195
206
  };
196
207
  const appendStdout = (chunk) => {
197
208
  const text = chunk.toString();
@@ -226,12 +237,21 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
226
237
  await writeRecord(recordDirectory, {
227
238
  task: task.id,
228
239
  hash: task.hash,
229
- verdict: judgeOperationalFailure(1, null),
240
+ verdict: judgeOperationalFailure(1, null, { kind: 'executor', message }),
230
241
  peakBytes: null,
231
242
  });
232
243
  }
233
244
  return { success: false, terminalOutput: outputChunks.join('') };
234
245
  };
246
+ let awakeNow = () => performance.now();
247
+ if (idleTimeoutMs !== undefined) {
248
+ try {
249
+ awakeNow = await openAwakeClock();
250
+ }
251
+ catch (error) {
252
+ return refuseUnstarted(`bounded-exec: cannot read its awake idle clock: ${String(error)}\n`);
253
+ }
254
+ }
235
255
  // The fixture pair names one segment: a fixture with no build budget, or a budget with no fixture, is a
236
256
  // target that does not say what it runs, so it does not run. Units are counted at the end of that segment.
237
257
  if ((options.fixtureCommand === undefined) !== (options.fixtureTimeoutMs === undefined) ||
@@ -399,6 +419,7 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
399
419
  parallelism: chargeParallelism,
400
420
  alone: attempt.alone,
401
421
  task: context.record?.task.id,
422
+ gated: demand.kind === 'nextest' && process.env[TEST_GATE_SERVED_ENV] === '1',
402
423
  });
403
424
  for (const said of parent.said)
404
425
  appendStderr(said);
@@ -409,16 +430,26 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
409
430
  // With mbx the leaf's admission is mbx's one pool, not a parent's tokens. A leaf mbx does not admit itself asks for
410
431
  // its runner's count, a maximum, and its runner runs on the cores it is granted, which only the command learns
411
432
  // (specs/cowshed/05_gateway.md, "Parallelism inside a job"). No grant and no sub-pool beside it.
412
- const leaf = mbx === null ? null : mbxLeaf(mbx, demand, requested);
433
+ const leaf = mbx === null ? null : mbxLeaf(mbx, demand, requested, process.env[TEST_GATE_SERVED_ENV] === '1');
434
+ // A nextest leaf granted fewer of its parent pool's tokens than it asked takes the case gate where the run's mbx
435
+ // gates cases: its runner starts every thread of its ask, and a case that finds no token of the grant free asks the
436
+ // pool for one of its own (specs/cowshed/05_gateway.md, "A case starts on a core", Inside a hold).
437
+ const lender = grant !== null &&
438
+ inherited !== null &&
439
+ demand.kind === 'nextest' &&
440
+ grant.tokens < demand.want &&
441
+ process.env[TEST_GATE_SERVED_ENV] === '1'
442
+ ? { own: grant.tokens, pool: inherited }
443
+ : null;
413
444
  if (grant !== null) {
414
- sized = sizeRunner(demand, requested, grant.tokens);
445
+ sized = sizeRunner(demand, requested, grant.tokens, lender !== null);
415
446
  cpu = { tokens: grant.tokens, want: demand.want, waitedMs };
416
447
  for (const said of describeGrant(grant, { waitedMs, want: demand.want, runner: demand.kind, charge })) {
417
448
  appendStderr(said);
418
449
  }
419
450
  }
420
451
  else if (leaf?.kind === 'reserve') {
421
- sized = { command: sizeRunnerToGrant(leaf.demand, requested), env: {} };
452
+ sized = { command: sizeRunnerToGrant(leaf.demand, requested, leaf.gated), env: {} };
422
453
  }
423
454
  else if (leaf !== null) {
424
455
  sized = sizeRunner(demand, requested, demand.want);
@@ -429,6 +460,38 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
429
460
  let pool = null;
430
461
  let reservation = null;
431
462
  const releaseReservation = () => reservation?.release();
463
+ // A gated nextest leaf's case gate (case-gate.ts; specs/cowshed/05_gateway.md, "A case starts on a core"): served
464
+ // before the command starts, decided from the leaf's grant on, and closed once the command is gone, when every core a
465
+ // case still holds goes back to the pool and the leaf says what its cases ran on.
466
+ let caseGate = null;
467
+ let gated = null;
468
+ let caseGateClosed = null;
469
+ const closeCaseGate = () => {
470
+ caseGateClosed ??= (async () => {
471
+ const left = (await caseGate?.close()) ?? null;
472
+ if (left !== null)
473
+ appendStderr(`bounded-exec: ${left}\n`);
474
+ const decided = gated;
475
+ if (decided === null)
476
+ return;
477
+ decided.releaseAll();
478
+ const { own, lent, lentCoreMs, waitedCaseMs } = decided.cases.summary();
479
+ tell(`cowshed: ${grant !== null ? 'cpu-pool' : 'mbx reserve'} cases own=${own} lent=${lent} lent_core_s=${Math.round(lentCoreMs / 1000)} waited_case_s=${Math.round(waitedCaseMs / 1000)}\n`);
480
+ })();
481
+ return caseGateClosed;
482
+ };
483
+ // The run's progress channel (progress-channel.ts), served when the run has an idle bound for its records to restart,
484
+ // and closed once its command is gone. A channel that leaves its socket behind says so; the run's verdict stands.
485
+ let progressChannel = null;
486
+ let progressClosed = null;
487
+ const closeProgress = () => {
488
+ progressClosed ??= (async () => {
489
+ const left = (await progressChannel?.close()) ?? null;
490
+ if (left !== null)
491
+ appendStderr(`bounded-exec: ${left}\n`);
492
+ })();
493
+ return progressClosed;
494
+ };
432
495
  let cpuReleased = null;
433
496
  const releaseCpu = () => {
434
497
  cpuReleased ??= (async () => {
@@ -452,11 +515,12 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
452
515
  tell,
453
516
  step: (event) => stepTo(event),
454
517
  testing: () => isTesting(),
518
+ awakeNow,
455
519
  queueHost,
456
520
  });
457
521
  // Records the run's verdict when it has a task to record as. An `expiry` is set only when a bound ended the run.
458
522
  // Observation and ownership failures remain operational even after valid queue evidence; cancellation alone does not.
459
- const recordRun = async (exitCode, expiry, elapsedMs, operational, noSpace, lockWaits, peakBytes) => {
523
+ const recordRun = async (exitCode, expiry, elapsedMs, operational, noSpace, lockWaits, cpuRising, peakBytes) => {
460
524
  if (context.record === undefined) {
461
525
  return;
462
526
  }
@@ -477,8 +541,9 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
477
541
  }
478
542
  });
479
543
  const admissionQueue = queue.queueWait(reservation !== null);
480
- const verdict = operational || resultFailure !== null
481
- ? judgeOperationalFailure(exitCode, report)
544
+ const cause = operational ?? (resultFailure === null ? null : { kind: 'executor', message: resultFailure });
545
+ const verdict = cause !== null
546
+ ? judgeOperationalFailure(exitCode, report, cause)
482
547
  : judgeRun({
483
548
  exitCode,
484
549
  elapsedMs,
@@ -486,6 +551,7 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
486
551
  report,
487
552
  noSpace,
488
553
  lockWaits,
554
+ cpuRising,
489
555
  admissionQueue,
490
556
  });
491
557
  if (verdict.outcome === 'bound' && verdict.bound === 'test') {
@@ -504,9 +570,11 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
504
570
  const refuse = async (message) => {
505
571
  appendStderr(message);
506
572
  await releaseCpu();
573
+ await closeProgress();
574
+ await closeCaseGate();
507
575
  // The command never started, so its lease holds nothing of its own to keep.
508
576
  await releaseLease(false);
509
- await recordRun(1, null, Date.now() - asked, false, null, [], null);
577
+ await recordRun(1, null, Date.now() - asked, { kind: 'executor', message: message.trim() }, null, [], [], null);
510
578
  return { success: false, terminalOutput: outputChunks.join('') };
511
579
  };
512
580
  if (parentRefusal !== null) {
@@ -529,6 +597,52 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
529
597
  }
530
598
  pool = created.pool;
531
599
  }
600
+ // A running test reports its own progress on the run's progress channel, and each record restarts the idle bound as a
601
+ // finished test does; a record that is not one fails the run as a broken reporter. A run with no idle bound has
602
+ // nothing for a record to restart and is served none. Only the command learns the socket, from its environment, so
603
+ // these handlers run after it is spawned and its bounds are wired below.
604
+ if (idleTimeoutMs !== undefined) {
605
+ const served = await serveProgressChannel({
606
+ moved: () => {
607
+ if (fixtureDone)
608
+ stepTo({ kind: 'reported', at: awakeNow(), testing: isTesting() });
609
+ },
610
+ invalid: (detail) => {
611
+ if (lifecycleFailure !== null || state.expiry !== null || state.settled)
612
+ return;
613
+ lifecycleFailure = { kind: 'executor', message: `progress channel: ${detail}` };
614
+ appendStderr(`bounded-exec: ${lifecycleFailure.message}\n`);
615
+ termination = terminate();
616
+ },
617
+ });
618
+ if (!served.ok) {
619
+ return refuse(`bounded-exec: the run's progress channel could not be served (${served.error}); refusing to run the command\n`);
620
+ }
621
+ progressChannel = served.channel;
622
+ }
623
+ // A case that arrives before the leaf's grant has no core to take and no leaf to wait for: its connection is refused,
624
+ // and its shim fails it by name.
625
+ if (reserved?.leaf.gated === true || lender !== null) {
626
+ const served = await serveCaseGate({
627
+ arrived: (id) => gated?.cases.arrived(id) ?? false,
628
+ ended: (id) => gated?.cases.ended(id),
629
+ invalid: (detail) => {
630
+ if (lifecycleFailure !== null || state.expiry !== null || state.settled)
631
+ return;
632
+ lifecycleFailure = { kind: 'executor', message: `case gate: ${detail}` };
633
+ appendStderr(`bounded-exec: ${lifecycleFailure.message}\n`);
634
+ termination = terminate();
635
+ },
636
+ });
637
+ if (!served.ok) {
638
+ return refuse(`bounded-exec: the leaf's case gate could not be served (${served.error}); refusing to run the command\n`);
639
+ }
640
+ caseGate = served.gate;
641
+ if (lender !== null) {
642
+ const gate = served.gate;
643
+ gated = gateCases(lender.own, lastWalls, (id) => gate.start(id), (id, left) => lendParentToken(lender.pool, `nextest:${id}`, left));
644
+ }
645
+ }
532
646
  const native = directory === null ? { command: sized.command, env: {} } : withTestResultReport(sized.command);
533
647
  const command = directory === null
534
648
  ? withProgressReport(native.command)
@@ -545,24 +659,35 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
545
659
  ...(pool?.environment ?? {}),
546
660
  ...(await grantMemoryEnvironment(grant?.charged ?? null, process.env)),
547
661
  ...(leaf?.kind === 'cargo' ? { MBX_GRANT_FD: String(MBX_GRANT_FD) } : {}),
662
+ ...(progressChannel === null ? {} : { [PROGRESS_SOCKET_ENV]: progressChannel.socket }),
663
+ ...(caseGate === null ? {} : { [TEST_GATE_ENV]: caseGate.socket }),
548
664
  };
549
665
  // A fixture rides as segment 0 of the lifecycle. Without the channel (Windows) it is plain shell text ahead
550
- // of the command: the progress bound runs during it and its own ceiling does not. Under mbx a leaf that names its
551
- // units counts them at the end of that segment, under the fixture's ceiling, its fixture's own output on stderr, and
552
- // reserves after it (`withCountedReservation`).
666
+ // of the command: the progress bound runs during it and its own ceiling does not. Under mbx a reserved leaf's
667
+ // fixture runs before its reservation, its ceiling from spawn (`withReservationAfterFixture`); a leaf that names its
668
+ // units counts them at the end of that segment, its fixture's own output on stderr (`withCountedReservation`).
553
669
  const counted = process.platform !== 'win32' && leaf?.kind === 'reserve' && fixture !== null && options.unitsCommand !== undefined;
554
670
  const segment = fixture === null ? null : counted ? `{ ${fixture.command}; } >&2 && ${options.unitsCommand}` : fixture.command;
555
671
  const fixtureHooked = process.platform === 'win32' || segment === null ? null : withFixtureLifecycle(segment, command);
556
672
  const hooked = process.platform === 'win32' ? null : (fixtureHooked ?? withRunnerLifecycle(command));
557
673
  const lifecycle = hooked === null ? null : createRunnerLifecycle(hooked.runners);
558
674
  let lifecycleFailure = null;
675
+ /** Why the executor ended or failed the run whatever its command did, if it did: the verdict's operational cause. */
676
+ /** The cause the executor ended the command for, if it ended it (closures set `lifecycleFailure`). */
677
+ const endedFor = () => lifecycleFailure;
678
+ const operationalCause = () => endedFor() ??
679
+ (state.poolCloseError !== null
680
+ ? { kind: 'executor', message: `cpu pool did not close cleanly: ${state.poolCloseError}` }
681
+ : state.retireError !== null
682
+ ? { kind: 'executor', message: `could not retire the command's processes: ${state.retireError}` }
683
+ : null);
559
684
  const isTesting = () => lifecycleFailure !== null || (lifecycle?.isTesting() ?? true);
560
685
  const startedAt = Date.now();
561
686
  // With a supervisor, `cowshed subtree-root` asks it for a handle on itself, writes the answer to the handle
562
687
  // descriptor, and execs `/bin/sh -c command` in the same process: the command's pid, group and life are unchanged.
563
688
  const unadmitted = hooked?.command ?? (segment === null ? command : `${segment} && ${command}`);
564
- const shellCommand = leaf?.kind === 'reserve' && counted && fixtureHooked !== null
565
- ? withCountedReservation(fixtureHooked.fixture, fixtureHooked.main, leaf)
689
+ const shellCommand = leaf?.kind === 'reserve' && fixtureHooked !== null
690
+ ? (counted ? withCountedReservation : withReservationAfterFixture)(fixtureHooked.fixture, fixtureHooked.main, leaf)
566
691
  : withMbxLeaf(unadmitted, leaf);
567
692
  // Only leaves known to be scheduled can exclude zero-hold time. Others keep lifecycle hang protection.
568
693
  const queued = mbxIdleGated(leaf, sized.command, { ...process.env, ...env });
@@ -644,14 +769,20 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
644
769
  return answer;
645
770
  });
646
771
  }
647
- // Progress is a finished test, not output: a runner's SLOW warnings and a hung test's heartbeat keep printing.
772
+ // Progress in the output is a finished test, never output itself: a runner's SLOW warnings and a hung test's heartbeat
773
+ // keep printing. A running test reports its own progress on the progress channel instead.
648
774
  const progress = createRunProgress(command, resultWriter === null
649
775
  ? undefined
650
- : (result) => {
776
+ : (closed) => {
777
+ // nextest times a case from its shim's start: the wait at the gate is taken off its wall and named beside it.
778
+ const waitMs = gated?.cases.waitedMs(closed.id) ?? 0;
779
+ const result = closed.durationMs === null || waitMs === 0
780
+ ? closed
781
+ : { ...closed, durationMs: runWallMs(closed.durationMs, waitMs), waitMs };
651
782
  const error = resultWriter?.append(result) ?? null;
652
783
  if (error !== null && resultFailure === null) {
653
784
  resultFailure = error;
654
- lifecycleFailure ??= error;
785
+ lifecycleFailure ??= { kind: 'executor', message: error };
655
786
  appendStderr(`bounded-exec: ${error}\n`);
656
787
  termination = terminate();
657
788
  }
@@ -670,20 +801,20 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
670
801
  let fixtureDone = fixture === null || lifecycle === null;
671
802
  // Whether a segment past the fixture ever announced: without one, the shell's exit is the fixture's.
672
803
  let mainStarted = fixture === null || lifecycle === null;
673
- let captureTimer;
804
+ let cancelCaptureTimer;
674
805
  let idleCapture;
675
806
  const cancelCapture = () => {
676
- clearTimeout(captureTimer);
807
+ cancelCaptureTimer?.();
677
808
  idleCapture?.controller.abort();
678
809
  };
679
- const scheduleCapture = (expiresAtMs) => {
810
+ const scheduleCapture = (awakeExpiresAtMs) => {
680
811
  const capture = {
681
- expiresAtMs,
812
+ scheduledExpiryAtMs: Date.now() + Math.ceil(Math.max(0, awakeExpiresAtMs - awakeNow())),
682
813
  controller: new AbortController(),
683
814
  result: { kind: 'pending', startedAtMs: null },
684
815
  };
685
816
  idleCapture = capture;
686
- captureTimer = setTimeout(() => {
817
+ cancelCaptureTimer = atClockDeadline(awakeExpiresAtMs - HANG_CAPTURE_MS, awakeNow, () => {
687
818
  if (state.settled || state.expiry !== null || idleCapture !== capture)
688
819
  return;
689
820
  const group = child.pid;
@@ -692,20 +823,23 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
692
823
  return;
693
824
  }
694
825
  idleCapture.result = { kind: 'pending', startedAtMs: Date.now() };
826
+ capture.scheduledExpiryAtMs = Date.now() + Math.ceil(Math.max(0, awakeExpiresAtMs - awakeNow()));
695
827
  void runnerPid
696
828
  .then((pid) => {
829
+ if (capture.controller.signal.aborted || awakeNow() >= awakeExpiresAtMs)
830
+ return;
697
831
  if (pid === null)
698
832
  throw new Error('runner PID unavailable: its waiter named no command; no holder conclusion');
699
833
  return captureDiagnostics({
700
834
  pid,
701
835
  pgid: process.platform === 'win32' ? null : group,
702
- expiresAtMs: capture.expiresAtMs,
703
- deadlineMs: Math.max(0, Math.min(HANG_CAPTURE_MS, capture.expiresAtMs - Date.now())),
836
+ awakeExpiresAtMs,
837
+ awakeNow,
704
838
  signal: capture.controller.signal,
705
839
  });
706
840
  })
707
841
  .then((snapshot) => {
708
- if (idleCapture === capture)
842
+ if (snapshot !== undefined && idleCapture === capture)
709
843
  idleCapture.result = { kind: 'captured', snapshot };
710
844
  }, (error) => {
711
845
  if (idleCapture === capture) {
@@ -715,7 +849,7 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
715
849
  };
716
850
  }
717
851
  });
718
- }, Math.max(0, expiresAtMs - Date.now() - HANG_CAPTURE_MS));
852
+ });
719
853
  };
720
854
  const expiredAs = (kind) => kind === 'idle'
721
855
  ? { kind, limitMs: idleTimeoutMs ?? 0, tests: progress.inFlight() }
@@ -723,26 +857,30 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
723
857
  ? { kind, limitMs: fixture?.timeoutMs ?? 0 }
724
858
  : { kind, limitMs: timeoutMs ?? 0 };
725
859
  let checkingIdle = false;
860
+ const nextBound = () => earliestBound(checkingIdle ? { ...bounds, idle: null } : bounds, awakeNow(), Date.now());
726
861
  const armBoundTimer = () => {
727
862
  clearTimeout(boundTimer);
728
863
  if (state.settled || state.expiry !== null)
729
864
  return;
730
- const next = earliestBound(checkingIdle ? { ...bounds, idle: null } : bounds);
865
+ const next = nextBound();
731
866
  if (next !== null) {
732
- boundTimer = setTimeout(() => {
733
- void confirmExpiry(next.kind, next.at);
734
- }, Math.max(0, next.at - Date.now()));
867
+ if (next.kind === 'idle' && idleCapture !== undefined && next.remainingMs > 0) {
868
+ idleCapture.scheduledExpiryAtMs = Date.now() + Math.ceil(next.remainingMs);
869
+ }
870
+ boundTimer = setTimeout(() => void confirmExpiry(), Math.ceil(Math.max(0, next.remainingMs)));
735
871
  }
736
872
  };
737
- const confirmExpiry = async (kind, at) => {
873
+ const confirmExpiry = async () => {
738
874
  if (state.settled || state.expiry !== null)
739
875
  return;
740
- const scheduled = earliestBound(checkingIdle ? { ...bounds, idle: null } : bounds);
741
- if (scheduled?.kind !== kind || scheduled.at !== at)
876
+ const scheduled = nextBound();
877
+ if (scheduled === null)
742
878
  return;
743
- if (kind === 'idle' && leaf?.kind === 'cargo') {
744
- if (checkingIdle)
745
- return;
879
+ if (scheduled.remainingMs > 0) {
880
+ armBoundTimer();
881
+ return;
882
+ }
883
+ if (scheduled.kind === 'idle' && leaf?.kind === 'cargo') {
746
884
  checkingIdle = true;
747
885
  armBoundTimer();
748
886
  try {
@@ -756,8 +894,8 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
756
894
  }
757
895
  if (state.settled || state.expiry !== null)
758
896
  return;
759
- const next = earliestBound(bounds);
760
- if (next !== null && next.at <= Date.now())
897
+ const next = nextBound();
898
+ if (next !== null && next.remainingMs <= 0)
761
899
  expire(expiredAs(next.kind));
762
900
  else
763
901
  armBoundTimer();
@@ -766,11 +904,14 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
766
904
  if (state.settled || state.expiry !== null)
767
905
  return;
768
906
  const idleBefore = bounds.idle;
769
- bounds = stepBounds(bounds, {
907
+ const next = stepBounds(bounds, {
770
908
  ...(idleTimeoutMs === undefined ? {} : { idleMs: idleTimeoutMs }),
771
909
  ...(fixture === null || lifecycle === null ? {} : { fixtureMs: fixture.timeoutMs }),
772
910
  ...(timeoutMs === undefined ? {} : { totalMs: timeoutMs }),
773
911
  }, event);
912
+ if (next === bounds)
913
+ return;
914
+ bounds = next;
774
915
  if (bounds.idle !== idleBefore) {
775
916
  cancelCapture();
776
917
  idleCapture = undefined;
@@ -783,7 +924,7 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
783
924
  // through appendStderr as well, and re-arming on those would let a teardown
784
925
  // message extend the very bound that just fired.
785
926
  const updateIdle = (finished, wasTesting) => {
786
- stepTo({ kind: 'progress', at: Date.now(), testing: isTesting(), wasTesting, finished });
927
+ stepTo({ kind: 'progress', at: awakeNow(), testing: isTesting(), wasTesting, finished });
787
928
  };
788
929
  const scanProgress = (stream, text) => {
789
930
  if (!fixtureDone)
@@ -791,7 +932,7 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
791
932
  updateIdle(progress.scan(stream, text), isTesting());
792
933
  const error = progress.error();
793
934
  if (error !== null && lifecycleFailure === null) {
794
- lifecycleFailure = error;
935
+ lifecycleFailure = { kind: 'executor', message: error };
795
936
  appendStderr(`bounded-exec: ${error}\n`);
796
937
  termination = terminate();
797
938
  }
@@ -803,7 +944,7 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
803
944
  const failLifecycle = (detail, wasTesting = isTesting()) => {
804
945
  if (lifecycleFailure !== null || state.expiry !== null)
805
946
  return;
806
- lifecycleFailure = detail;
947
+ lifecycleFailure = { kind: 'executor', message: `runner lifecycle failed: ${detail}` };
807
948
  appendStderr(`bounded-exec: runner lifecycle failed: ${detail}\n`);
808
949
  updateIdle(false, wasTesting);
809
950
  };
@@ -881,10 +1022,12 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
881
1022
  queue.stop();
882
1023
  return killChildTree();
883
1024
  })
1025
+ .finally(closeProgress)
1026
+ .finally(closeCaseGate)
884
1027
  .finally(releaseCpu)
885
1028
  .finally(() => releaseLease(true))
886
1029
  .finally(releaseReservation)
887
- .finally(async () => recordRun(signalToExitCode(signal), state.expiry, Date.now() - startedAt, state.poolCloseError !== null || lifecycleFailure !== null || state.retireError !== null, state.noSpace === null ? null : await state.noSpace, state.lockWaits, null))
1030
+ .finally(async () => recordRun(signalToExitCode(signal), state.expiry, Date.now() - startedAt, operationalCause(), state.noSpace === null ? null : await state.noSpace, state.lockWaits, state.cpuRising, null))
888
1031
  .finally(() => process.kill(process.pid, signal));
889
1032
  };
890
1033
  const onSigint = () => onTerminationSignal('SIGINT');
@@ -930,7 +1073,7 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
930
1073
  clearTimeout(boundTimer);
931
1074
  const elapsedMs = Date.now() - startedAt;
932
1075
  appendStderr(bound.kind === 'idle'
933
- ? `\nCommand made no progress: no test finished for ${bound.limitMs}ms (idleTimeoutMs=${bound.limitMs}) after ${elapsedMs}ms of runtime; ${bound.tests.length > 0
1076
+ ? `\nCommand made no progress: no test finished or reported progress for ${bound.limitMs}ms (idleTimeoutMs=${bound.limitMs}) after ${elapsedMs}ms of runtime; ${bound.tests.length > 0
934
1077
  ? `still running: ${bound.tests.join(', ')}`
935
1078
  : 'the runner named no test still running'} (cwd=${cwd}): ${command}\n${queue.unconfirmed === null
936
1079
  ? ''
@@ -941,7 +1084,8 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
941
1084
  if (bound.kind === 'idle') {
942
1085
  const observed = idleCapture?.result;
943
1086
  state.lockWaits = observed?.kind === 'captured' ? observed.snapshot.lockWaits : [];
944
- appendStderr(`Hang diagnostics: runner pid=${runner ?? 'unavailable'} pgid=${process.platform === 'win32' ? 'unavailable' : (child.pid ?? 'unavailable')} expiryAtMs=${Date.now()} scheduledExpiryAtMs=${idleCapture?.expiresAtMs ?? 'unavailable'}\n`);
1087
+ state.cpuRising = observed?.kind === 'captured' ? observed.snapshot.cpuRising : [];
1088
+ appendStderr(`Hang diagnostics: runner pid=${runner ?? 'unavailable'} pgid=${process.platform === 'win32' ? 'unavailable' : (child.pid ?? 'unavailable')} expiryAtMs=${Date.now()} scheduledExpiryAtMs=${idleCapture?.scheduledExpiryAtMs ?? 'unavailable'}\n`);
945
1089
  if (observed?.kind === 'captured') {
946
1090
  appendStderr(`Hang diagnostics: outcome=${observed.snapshot.outcome} capturedAtMs=${observed.snapshot.capturedAtMs} actualExpiryOffsetMs=${Date.now() - observed.snapshot.capturedAtMs}\n${observed.snapshot.text}`);
947
1091
  }
@@ -1017,24 +1161,29 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
1017
1161
  // fire on a command the fixture already handed over to.
1018
1162
  stepTo({
1019
1163
  kind: 'spawn',
1020
- at: Date.now(),
1164
+ at: awakeNow(),
1165
+ wallAt: Date.now(),
1021
1166
  testing: isTesting(),
1022
1167
  fixture: fixture !== null && lifecycle !== null,
1023
- fixtureFirst: counted && lifecycle !== null,
1168
+ fixtureFirst: leaf?.kind === 'reserve' && fixtureHooked !== null,
1024
1169
  queued,
1025
1170
  });
1026
1171
  if (leaf !== null) {
1027
1172
  const channel = child.stdio.at(MBX_GRANT_FD);
1028
1173
  if (!(channel instanceof Duplex))
1029
1174
  throw new Error('an mbx leaf requires its mbx descriptor');
1030
- const failGrant = (detail) => {
1175
+ const descriptorFailure = (detail) => ({
1176
+ kind: 'executor',
1177
+ message: `mbx grant descriptor: ${detail}`,
1178
+ });
1179
+ const failGrant = (cause) => {
1031
1180
  if (lifecycleFailure !== null || state.expiry !== null || state.settled)
1032
1181
  return;
1033
- lifecycleFailure ??= `mbx grant descriptor: ${detail}`;
1034
- appendStderr(`bounded-exec: ${lifecycleFailure}\n`);
1182
+ lifecycleFailure = cause;
1183
+ appendStderr(`bounded-exec: ${cause.message}\n`);
1035
1184
  termination = terminate();
1036
1185
  };
1037
- channel.on('error', (error) => failGrant(error.message));
1186
+ channel.on('error', (error) => failGrant(descriptorFailure(error.message)));
1038
1187
  if (reserved !== null) {
1039
1188
  // The shell asks for its cores in one line and reads the count granted (mbx-pool.ts, `reservation`). A refusal
1040
1189
  // ends the descriptor unanswered: the shell's read fails, and nothing runs.
@@ -1043,7 +1192,7 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
1043
1192
  channel.setEncoding('utf8');
1044
1193
  channel.on('data', (text) => {
1045
1194
  if (reservation !== null) {
1046
- failGrant(`the command asked for cores a second time: ${JSON.stringify(text)}`);
1195
+ failGrant(descriptorFailure(`the command asked for cores a second time: ${JSON.stringify(text)}`));
1047
1196
  return;
1048
1197
  }
1049
1198
  asked += text;
@@ -1053,17 +1202,26 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
1053
1202
  const line = asked.slice(0, end);
1054
1203
  const cpus = askedCores(line, reservedLeaf.demand.want);
1055
1204
  if (cpus === null) {
1056
- failGrant(`the command asked for ${JSON.stringify(line)} cores, not 1 to ${reservedLeaf.demand.want}`);
1205
+ failGrant(descriptorFailure(`the command asked for ${JSON.stringify(line)} cores, not 1 to ${reservedLeaf.demand.want}`));
1057
1206
  channel.end();
1058
1207
  return;
1059
1208
  }
1060
1209
  queue.waiting();
1061
- tell(`cowshed: mbx reserve wait start cpus=${cpus} priority=${reservedLeaf.priority}\n`);
1062
- const held = reserveCores(endpoint, cpus, reservedLeaf.priority);
1210
+ // A whole-suite test leaf names the fewest threads at which its recorded walls keep a test finishing within
1211
+ // its own no-progress bound (specs/cowshed/05_gateway.md, "A minimum for a whole suite").
1212
+ const min = idleTimeoutMs === undefined ? 1 : suiteMinimum([...lastWalls.values()], cpus, idleTimeoutMs);
1213
+ const named = min > 1 ? ` min=${min}` : '';
1214
+ tell(`cowshed: mbx reserve wait start cpus=${cpus}${named} priority=${reservedLeaf.priority}\n`);
1215
+ const held = reserveCores(endpoint, cpus, reservedLeaf.priority, min);
1063
1216
  reservation = held;
1064
1217
  void held.granted.then((granted) => {
1065
1218
  if (!granted.ok) {
1066
- failGrant(`mbx did not grant the reservation: ${granted.error}`);
1219
+ failGrant({
1220
+ kind: 'reservation-refused',
1221
+ endpoint: granted.refusal.endpoint,
1222
+ errno: granted.refusal.errno,
1223
+ message: `mbx did not grant the reservation: ${granted.refusal.message}`,
1224
+ });
1067
1225
  channel.end();
1068
1226
  return;
1069
1227
  }
@@ -1072,8 +1230,15 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
1072
1230
  return;
1073
1231
  queue.admitted = true;
1074
1232
  tell(`cowshed: mbx reserve granted elapsed=${Date.now() - startedAt}ms priority=${reservedLeaf.priority}\n`);
1233
+ if (caseGate !== null) {
1234
+ const gate = caseGate;
1235
+ gated = gateCases(granted.cpus, lastWalls, (id) => gate.start(id), (id) => {
1236
+ const held = reserveCores(endpoint, 1, 'low', 1, `nextest:${id}`);
1237
+ return { granted: held.granted.then((answer) => answer.ok), release: () => held.release() };
1238
+ });
1239
+ }
1075
1240
  channel.end(`${granted.cpus}\n`);
1076
- stepTo({ kind: 'granted', at: Date.now(), testing: isTesting(), fixture: !fixtureDone });
1241
+ stepTo({ kind: 'granted', at: awakeNow(), wallAt: Date.now(), testing: isTesting(), fixture: !fixtureDone });
1077
1242
  void granted.lost.then((why) => {
1078
1243
  if (state.settled || cpuReleased !== null)
1079
1244
  return;
@@ -1093,13 +1258,19 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
1093
1258
  said = true;
1094
1259
  tell(`cowshed: mbx cargo granted elapsed=${Date.now() - startedAt}ms\n`);
1095
1260
  }
1096
- stepTo({ kind: 'granted', at: Date.now(), testing: isTesting(), fixture: !fixtureDone });
1261
+ stepTo({
1262
+ kind: 'granted',
1263
+ at: awakeNow(),
1264
+ wallAt: Date.now(),
1265
+ testing: isTesting(),
1266
+ fixture: !fixtureDone,
1267
+ });
1097
1268
  }
1098
1269
  else if (byte === 114) {
1099
- stepTo({ kind: 'released', at: Date.now() });
1270
+ stepTo({ kind: 'released', at: awakeNow() });
1100
1271
  }
1101
1272
  else {
1102
- failGrant(`unexpected byte ${byte}`);
1273
+ failGrant(descriptorFailure(`unexpected byte ${byte}`));
1103
1274
  return;
1104
1275
  }
1105
1276
  }
@@ -1132,16 +1303,23 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
1132
1303
  clearTimeout(boundTimer);
1133
1304
  cancelCapture();
1134
1305
  removeSignalHandlers();
1135
- // The runner is gone. Its sub-pool closes first, so no descendant still draws on tokens the grant returns;
1136
- // then the grant goes back to its parent or the host.
1306
+ // The runner is gone, and with it every test that could report on the run's progress channel. Its sub-pool closes
1307
+ // first, so no descendant still draws on tokens the grant returns; then the grant goes back to its parent or the host.
1308
+ await closeProgress();
1309
+ await closeCaseGate();
1137
1310
  await releaseCpu();
1138
1311
  const code = exitCode ?? signalToExitCode(exitSignal);
1139
1312
  if (code !== 0 && state.expiry === null) {
1140
- // No bound of this executor ended it, so a signal it died of came from outside (the kernel's OOM killer sends
1141
- // SIGKILL): the signal is named, never folded into a bare status.
1142
- const ended = exitCode === null && exitSignal !== null
1143
- ? `was killed by ${exitSignal} (status ${code})`
1144
- : `exited with status ${code}`;
1313
+ // The executor ended it for a cause of its own, which is said; otherwise no bound of this executor ended it, so a
1314
+ // signal it died of came from outside (the kernel's OOM killer sends SIGKILL): the signal is named, never folded
1315
+ // into a bare status.
1316
+ const signalled = exitCode === null && exitSignal !== null;
1317
+ const endedBy = endedFor();
1318
+ const ended = endedBy !== null
1319
+ ? `was ended by bounded-exec because ${endedBy.message} (${signalled ? `${exitSignal}, ` : ''}status ${code})`
1320
+ : signalled
1321
+ ? `was killed by ${exitSignal} (status ${code})`
1322
+ : `exited with status ${code}`;
1145
1323
  appendStderr(fixture !== null && !mainStarted
1146
1324
  ? `Fixture command ${ended}: ${segment ?? fixture.command}\n`
1147
1325
  : `Command ${ended}: ${command}\n`);
@@ -1170,10 +1348,15 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
1170
1348
  appendStderr(`bounded-exec: this run's memory peak was not measured (${reported.reason}); its record names none\n`);
1171
1349
  }
1172
1350
  const peakBytes = reported?.kind === 'measured' ? reported.bytes : null;
1351
+ // A Go run sums nothing itself, and a reader that keeps part of a long output cannot count it: the sum ends the output.
1352
+ const goRun = progress.goSummary();
1353
+ if (goRun !== null) {
1354
+ appendStdout(goRunSummaryText(goRun, declaredGoTests(goRun.failedTests, cwd, context.root, mergeEnv(lease?.directory ?? ordinaryTemp, env))));
1355
+ }
1173
1356
  // Broken resource ownership, a lost native lifecycle, or leftovers that could not be retired fail even if a shell
1174
1357
  // branch later exited 0: its verdict is an ordinary operational failure, never a successful or silently unbounded run.
1175
- const operationalFailure = state.poolCloseError !== null || lifecycleFailure !== null || state.retireError !== null;
1176
- await recordRun(code, state.expiry, Date.now() - startedAt, operationalFailure, seen, state.lockWaits, peakBytes);
1358
+ const operationalFailure = operationalCause();
1359
+ await recordRun(code, state.expiry, Date.now() - startedAt, operationalFailure, seen, state.lockWaits, state.cpuRising, peakBytes);
1177
1360
  const killedForMemory = killedFromOutside({
1178
1361
  sentSignal: state.sentSignal,
1179
1362
  bounded: state.expiry !== null,
@@ -1181,7 +1364,7 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
1181
1364
  code,
1182
1365
  });
1183
1366
  return {
1184
- success: !operationalFailure && resultFailure === null && state.expiry === null && code === 0,
1367
+ success: operationalFailure === null && resultFailure === null && state.expiry === null && code === 0,
1185
1368
  terminalOutput: outputChunks.join(''),
1186
1369
  ...(killedForMemory ? { killedForMemory: true } : {}),
1187
1370
  };
@@ -1199,19 +1382,6 @@ function resolveCwd(cwd, root) {
1199
1382
  }
1200
1383
  return isAbsolute(cwd) ? cwd : join(root, cwd);
1201
1384
  }
1202
- function buildCommand(options) {
1203
- const parts = [options.command];
1204
- if (Array.isArray(options.args)) {
1205
- parts.push(...options.args);
1206
- }
1207
- else if (options.args) {
1208
- parts.push(options.args);
1209
- }
1210
- if (options.forwardAllArgs !== false && options.__unparsed__?.length) {
1211
- parts.push(...options.__unparsed__);
1212
- }
1213
- return parts.join(' ');
1214
- }
1215
1385
  /** Owned scratch is the command's TMPDIR unless the target names one itself. */
1216
1386
  function mergeEnv(directory, env) {
1217
1387
  const inherited = withoutPoolEnv(process.env);