@smoothbricks/nx-plugin 0.7.18 → 0.7.19

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (143) hide show
  1. package/README.md +160 -99
  2. package/dist/.build-inputs.json +35 -29
  3. package/dist/built-output-packages.js +1 -1
  4. package/dist/cargo-closure-input.js +1 -1
  5. package/dist/cargo-source-hash.d.ts +3 -21
  6. package/dist/cargo-source-hash.d.ts.map +1 -1
  7. package/dist/cargo-source-hash.js +4 -91
  8. package/dist/cargo-toolchain-paths.d.ts +2 -2
  9. package/dist/cargo-toolchain-paths.js +2 -2
  10. package/dist/cargo-workspace.d.ts.map +1 -1
  11. package/dist/cargo-workspace.js +2 -1
  12. package/dist/executors/bounded-exec/admission-queue.d.ts +3 -1
  13. package/dist/executors/bounded-exec/admission-queue.d.ts.map +1 -1
  14. package/dist/executors/bounded-exec/admission-queue.js +9 -10
  15. package/dist/executors/bounded-exec/awake-clock.d.ts +5 -0
  16. package/dist/executors/bounded-exec/awake-clock.d.ts.map +1 -0
  17. package/dist/executors/bounded-exec/awake-clock.js +22 -0
  18. package/dist/executors/bounded-exec/bounds.d.ts +7 -5
  19. package/dist/executors/bounded-exec/bounds.d.ts.map +1 -1
  20. package/dist/executors/bounded-exec/bounds.js +14 -11
  21. package/dist/executors/bounded-exec/case-gate.d.ts +114 -0
  22. package/dist/executors/bounded-exec/case-gate.d.ts.map +1 -0
  23. package/dist/executors/bounded-exec/case-gate.js +342 -0
  24. package/dist/executors/bounded-exec/cpu-pool-thread.d.ts +1 -0
  25. package/dist/executors/bounded-exec/cpu-pool-thread.d.ts.map +1 -1
  26. package/dist/executors/bounded-exec/cpu-pool-thread.js +2 -2
  27. package/dist/executors/bounded-exec/cpu-pool.d.ts +28 -1
  28. package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -1
  29. package/dist/executors/bounded-exec/cpu-pool.js +102 -16
  30. package/dist/executors/bounded-exec/endpoint-life.d.ts +146 -0
  31. package/dist/executors/bounded-exec/endpoint-life.d.ts.map +1 -0
  32. package/dist/executors/bounded-exec/endpoint-life.js +228 -0
  33. package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
  34. package/dist/executors/bounded-exec/executor.js +207 -69
  35. package/dist/executors/bounded-exec/gateway-lease.d.ts +1 -1
  36. package/dist/executors/bounded-exec/gateway-lease.d.ts.map +1 -1
  37. package/dist/executors/bounded-exec/gateway-lease.js +8 -1
  38. package/dist/executors/bounded-exec/go-run-summary.d.ts +51 -0
  39. package/dist/executors/bounded-exec/go-run-summary.d.ts.map +1 -0
  40. package/dist/executors/bounded-exec/go-run-summary.js +98 -0
  41. package/dist/executors/bounded-exec/hang-diagnostics.d.ts +3 -3
  42. package/dist/executors/bounded-exec/hang-diagnostics.d.ts.map +1 -1
  43. package/dist/executors/bounded-exec/hang-diagnostics.js +66 -48
  44. package/dist/executors/bounded-exec/lease-wait.d.ts +1 -0
  45. package/dist/executors/bounded-exec/lease-wait.d.ts.map +1 -1
  46. package/dist/executors/bounded-exec/lease-wait.js +1 -0
  47. package/dist/executors/bounded-exec/mbx-pool.d.ts +28 -7
  48. package/dist/executors/bounded-exec/mbx-pool.d.ts.map +1 -1
  49. package/dist/executors/bounded-exec/mbx-pool.js +30 -9
  50. package/dist/executors/bounded-exec/mbx-reservations.d.ts +8 -5
  51. package/dist/executors/bounded-exec/mbx-reservations.d.ts.map +1 -1
  52. package/dist/executors/bounded-exec/mbx-reservations.js +187 -283
  53. package/dist/executors/bounded-exec/parent-grant.d.ts +2 -0
  54. package/dist/executors/bounded-exec/parent-grant.d.ts.map +1 -1
  55. package/dist/executors/bounded-exec/parent-grant.js +1 -0
  56. package/dist/executors/bounded-exec/pool-admission.d.ts +29 -3
  57. package/dist/executors/bounded-exec/pool-admission.d.ts.map +1 -1
  58. package/dist/executors/bounded-exec/pool-admission.js +47 -1
  59. package/dist/executors/bounded-exec/progress.d.ts +3 -0
  60. package/dist/executors/bounded-exec/progress.d.ts.map +1 -1
  61. package/dist/executors/bounded-exec/progress.js +31 -1
  62. package/dist/executors/bounded-exec/reservation-connection.d.ts +90 -0
  63. package/dist/executors/bounded-exec/reservation-connection.d.ts.map +1 -0
  64. package/dist/executors/bounded-exec/reservation-connection.js +248 -0
  65. package/dist/executors/bounded-exec/run-cpu-pool.d.ts.map +1 -1
  66. package/dist/executors/bounded-exec/run-cpu-pool.js +3 -0
  67. package/dist/executors/bounded-exec/runner-sizing.d.ts +9 -5
  68. package/dist/executors/bounded-exec/runner-sizing.d.ts.map +1 -1
  69. package/dist/executors/bounded-exec/runner-sizing.js +13 -9
  70. package/dist/executors/bounded-exec/schema.d.ts +2 -2
  71. package/dist/executors/bounded-exec/suite-minimum.d.ts +21 -0
  72. package/dist/executors/bounded-exec/suite-minimum.d.ts.map +1 -0
  73. package/dist/executors/bounded-exec/suite-minimum.js +85 -0
  74. package/dist/executors/bounded-exec/test-result-writer.d.ts +2 -0
  75. package/dist/executors/bounded-exec/test-result-writer.d.ts.map +1 -1
  76. package/dist/executors/bounded-exec/test-result-writer.js +1 -0
  77. package/dist/executors/bounded-exec/test-results.d.ts +5 -0
  78. package/dist/executors/bounded-exec/test-results.d.ts.map +1 -1
  79. package/dist/executors/bounded-exec/test-results.js +2 -2
  80. package/dist/executors/bounded-exec/verdict.d.ts +26 -4
  81. package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
  82. package/dist/executors/bounded-exec/verdict.js +26 -5
  83. package/dist/executors/cargo-zigbuild/executor.d.ts +10 -8
  84. package/dist/executors/cargo-zigbuild/executor.d.ts.map +1 -1
  85. package/dist/executors/cargo-zigbuild/executor.js +32 -19
  86. package/dist/index.d.ts.map +1 -1
  87. package/dist/index.js +105 -73
  88. package/dist/managed-files/ci-workflow.d.ts +4 -1
  89. package/dist/managed-files/ci-workflow.d.ts.map +1 -1
  90. package/dist/managed-files/ci-workflow.js +9 -14
  91. package/dist/managed-files/files.d.ts +3 -3
  92. package/dist/managed-files/files.js +3 -3
  93. package/dist/task-results.js +2 -1
  94. package/dist/workspace-config-policy.d.ts.map +1 -1
  95. package/dist/workspace-config-policy.js +135 -42
  96. package/executors.json +2 -0
  97. package/managed/raw/envrc +24 -0
  98. package/managed/raw/patches/nx@24.0.3.patch +1179 -133
  99. package/managed/raw/patches/ttsc@0.37.1.patch +1055 -0
  100. package/managed/raw/tooling/direnv/developer-links.ts +105 -11
  101. package/managed/raw/tooling/direnv/devenv.smoo.nix +20 -19
  102. package/package.json +67 -11
  103. package/runtime/nx-publish/.build-inputs.json +2 -2
  104. package/runtime/nx-publish/cache_adapter.js +293 -3
  105. package/runtime/nx-publish/nx_publish.js +36 -14
  106. package/src/built-output-packages.ts +1 -1
  107. package/src/cargo-closure-input.ts +1 -1
  108. package/src/cargo-source-hash.ts +16 -132
  109. package/src/cargo-toolchain-paths.ts +2 -2
  110. package/src/cargo-workspace.ts +1 -2
  111. package/src/executors/bounded-exec/admission-queue.ts +11 -10
  112. package/src/executors/bounded-exec/awake-clock.ts +20 -0
  113. package/src/executors/bounded-exec/bounds.ts +28 -15
  114. package/src/executors/bounded-exec/case-gate.ts +365 -0
  115. package/src/executors/bounded-exec/cpu-pool-thread.ts +3 -3
  116. package/src/executors/bounded-exec/cpu-pool.ts +139 -12
  117. package/src/executors/bounded-exec/endpoint-life.ts +279 -0
  118. package/src/executors/bounded-exec/executor.ts +280 -106
  119. package/src/executors/bounded-exec/gateway-lease.ts +6 -0
  120. package/src/executors/bounded-exec/go-run-summary.ts +137 -0
  121. package/src/executors/bounded-exec/hang-diagnostics.ts +70 -52
  122. package/src/executors/bounded-exec/lease-wait.ts +1 -0
  123. package/src/executors/bounded-exec/mbx-pool.ts +36 -9
  124. package/src/executors/bounded-exec/mbx-reservations.ts +213 -182
  125. package/src/executors/bounded-exec/parent-grant.ts +3 -0
  126. package/src/executors/bounded-exec/pool-admission.ts +59 -2
  127. package/src/executors/bounded-exec/progress.ts +37 -2
  128. package/src/executors/bounded-exec/reservation-connection.ts +221 -0
  129. package/src/executors/bounded-exec/run-cpu-pool.ts +2 -0
  130. package/src/executors/bounded-exec/runner-sizing.ts +13 -8
  131. package/src/executors/bounded-exec/schema.json +1 -1
  132. package/src/executors/bounded-exec/schema.ts +2 -2
  133. package/src/executors/bounded-exec/suite-minimum.ts +81 -0
  134. package/src/executors/bounded-exec/test-result-writer.ts +3 -0
  135. package/src/executors/bounded-exec/test-results.ts +5 -0
  136. package/src/executors/bounded-exec/verdict.ts +39 -6
  137. package/src/executors/cargo-zigbuild/executor.ts +30 -19
  138. package/src/index.ts +112 -78
  139. package/src/managed-files/ci-workflow.ts +13 -17
  140. package/src/managed-files/files.ts +3 -3
  141. package/src/workspace-config-policy.ts +161 -43
  142. package/test-timings.json +157 -147
  143. package/managed/raw/patches/ttsc@0.36.3.patch +0 -1660
@@ -14,11 +14,22 @@ import {
14
14
  signalProcessGroup,
15
15
  } from '../../process-group.js';
16
16
  import { AdmissionObservation, type QueueHost } from './admission-queue.js';
17
+ import { atClockDeadline, openAwakeClock } from './awake-clock.js';
17
18
  import { type BoundEvent, type BoundKind, type Bounds, earliestBound, stepBounds, UNARMED } from './bounds.js';
19
+ import {
20
+ type CaseGateServer,
21
+ type GatedCases,
22
+ gateCases,
23
+ runWallMs,
24
+ serveCaseGate,
25
+ TEST_GATE_ENV,
26
+ TEST_GATE_SERVED_ENV,
27
+ } from './case-gate.js';
18
28
  import { DIAGNOSTIC_CONTEXT_ENV, DIAGNOSTIC_MODULE_ENV, grantMemoryEnvironment } from './child-exit-diagnostics.js';
19
- import { type CpuPool, createCpuPool } from './cpu-pool.js';
29
+ import { type CpuPool, createCpuPool, lendParentToken } from './cpu-pool.js';
20
30
  import type { CpuRise } from './cpu-rise.js';
21
31
  import { GATEWAY_SOCKET } from './gateway-lease.js';
32
+ import { declaredGoTests, goRunSummaryText } from './go-run-summary.js';
22
33
  import type { HangDiagnostics } from './hang-diagnostics.js';
23
34
  import { captureHangDiagnostics, HANG_CAPTURE_MS } from './hang-diagnostics.js';
24
35
  import { type LeafPeak, PEAK_FD, peakWaiter, readWaiterReport, underWaiter } from './leaf-peak.js';
@@ -31,6 +42,7 @@ import {
31
42
  mbxPool,
32
43
  withCountedReservation,
33
44
  withMbxLeaf,
45
+ withReservationAfterFixture,
34
46
  } from './mbx-pool.js';
35
47
  import { describeUnserved, RESERVATIONS_ENV, type Reservation, reserveCores, runEndpoint } from './mbx-reservations.js';
36
48
  import { killedFromOutside, type RunOutcome, runUntilNotKilledForMemory } from './memory-retry.js';
@@ -65,6 +77,7 @@ import { createRunnerLifecycle, withFixtureLifecycle, withRunnerLifecycle } from
65
77
  import { CPU_TOKENS_ENV, runnerDemand, sizeRunner, sizeRunnerToGrant } from './runner-sizing.js';
66
78
  import { checkRuntimePin, describeRuntimePinRefusal } from './runtime-pin.js';
67
79
  import { type BoundedExecOptions, boundedCommand } from './schema.js';
80
+ import { readLastWalls, suiteMinimum } from './suite-minimum.js';
68
81
  import { createTestResultWriter, type TestResultWriter } from './test-result-writer.js';
69
82
  import {
70
83
  type BoundedExecRecord,
@@ -78,6 +91,7 @@ import {
78
91
  judgeOperationalFailure,
79
92
  judgeRun,
80
93
  type NoSpaceSighting,
94
+ type OperationalCause,
81
95
  readJunitReport,
82
96
  taskDirectory,
83
97
  withJunitReport,
@@ -309,10 +323,14 @@ export async function runBoundedExec(
309
323
  // pool, read once before a run's clear removes them: a command run again after a kill is charged by the record the
310
324
  // killed run replaced, not by the peak it reached before it was killed.
311
325
  const lastPeak = directory !== null && inherited !== null ? await readLastPeak(directory) : null;
326
+ // The walls its last run recorded size the minimum its reservation names (suite-minimum.ts), read before the same
327
+ // clear removes them.
328
+ const lastWalls = directory !== null ? await readLastWalls(directory) : new Map<string, number>();
312
329
  return runUntilNotKilledForMemory(
313
330
  (alone) =>
314
331
  runCommand(options, context, killer, tempVolume, inherited, captureDiagnostics, owner, queueHost, {
315
332
  lastPeak,
333
+ lastWalls,
316
334
  alone,
317
335
  }),
318
336
  inherited !== null,
@@ -320,9 +338,13 @@ export async function runBoundedExec(
320
338
  );
321
339
  }
322
340
 
323
- /** What a run of the command is told that the run before it was not: the task's charge, and whether it runs alone. */
341
+ /**
342
+ * What a run of the command is told that the run before it was not: the task's charge, the walls its tests last took,
343
+ * and whether it runs alone.
344
+ */
324
345
  interface Attempt {
325
346
  readonly lastPeak: LastPeak | null;
347
+ readonly lastWalls: ReadonlyMap<string, number>;
326
348
  readonly alone: boolean;
327
349
  }
328
350
 
@@ -340,7 +362,7 @@ async function runCommand(
340
362
  ): Promise<BoundedExecResult> {
341
363
  const cwd = resolveCwd(options.cwd, context.root);
342
364
  const directory = context.record?.directory ?? null;
343
- const { lastPeak } = attempt;
365
+ const { lastPeak, lastWalls } = attempt;
344
366
  // A previous run's verdict and report go first, so neither can be read as this run's.
345
367
  if (directory !== null) {
346
368
  await clearTaskDirectory(directory);
@@ -398,13 +420,21 @@ async function runCommand(
398
420
  await writeRecord(recordDirectory, {
399
421
  task: task.id,
400
422
  hash: task.hash,
401
- verdict: judgeOperationalFailure(1, null),
423
+ verdict: judgeOperationalFailure(1, null, { kind: 'executor', message }),
402
424
  peakBytes: null,
403
425
  });
404
426
  }
405
427
  return { success: false, terminalOutput: outputChunks.join('') };
406
428
  };
407
429
 
430
+ let awakeNow = (): number => performance.now();
431
+ if (idleTimeoutMs !== undefined) {
432
+ try {
433
+ awakeNow = await openAwakeClock();
434
+ } catch (error) {
435
+ return refuseUnstarted(`bounded-exec: cannot read its awake idle clock: ${String(error)}\n`);
436
+ }
437
+ }
408
438
  // The fixture pair names one segment: a fixture with no build budget, or a budget with no fixture, is a
409
439
  // target that does not say what it runs, so it does not run. Units are counted at the end of that segment.
410
440
  if (
@@ -587,6 +617,7 @@ async function runCommand(
587
617
  parallelism: chargeParallelism,
588
618
  alone: attempt.alone,
589
619
  task: context.record?.task.id,
620
+ gated: demand.kind === 'nextest' && process.env[TEST_GATE_SERVED_ENV] === '1',
590
621
  });
591
622
  for (const said of parent.said) appendStderr(said);
592
623
  const { grant, refusal: parentRefusal, wait: parentWait } = parent;
@@ -596,15 +627,26 @@ async function runCommand(
596
627
  // With mbx the leaf's admission is mbx's one pool, not a parent's tokens. A leaf mbx does not admit itself asks for
597
628
  // its runner's count, a maximum, and its runner runs on the cores it is granted, which only the command learns
598
629
  // (specs/cowshed/05_gateway.md, "Parallelism inside a job"). No grant and no sub-pool beside it.
599
- const leaf = mbx === null ? null : mbxLeaf(mbx, demand, requested);
630
+ const leaf = mbx === null ? null : mbxLeaf(mbx, demand, requested, process.env[TEST_GATE_SERVED_ENV] === '1');
631
+ // A nextest leaf granted fewer of its parent pool's tokens than it asked takes the case gate where the run's mbx
632
+ // gates cases: its runner starts every thread of its ask, and a case that finds no token of the grant free asks the
633
+ // pool for one of its own (specs/cowshed/05_gateway.md, "A case starts on a core", Inside a hold).
634
+ const lender =
635
+ grant !== null &&
636
+ inherited !== null &&
637
+ demand.kind === 'nextest' &&
638
+ grant.tokens < demand.want &&
639
+ process.env[TEST_GATE_SERVED_ENV] === '1'
640
+ ? { own: grant.tokens, pool: inherited }
641
+ : null;
600
642
  if (grant !== null) {
601
- sized = sizeRunner(demand, requested, grant.tokens);
643
+ sized = sizeRunner(demand, requested, grant.tokens, lender !== null);
602
644
  cpu = { tokens: grant.tokens, want: demand.want, waitedMs };
603
645
  for (const said of describeGrant(grant, { waitedMs, want: demand.want, runner: demand.kind, charge })) {
604
646
  appendStderr(said);
605
647
  }
606
648
  } else if (leaf?.kind === 'reserve') {
607
- sized = { command: sizeRunnerToGrant(leaf.demand, requested), env: {} };
649
+ sized = { command: sizeRunnerToGrant(leaf.demand, requested, leaf.gated), env: {} };
608
650
  } else if (leaf !== null) {
609
651
  sized = sizeRunner(demand, requested, demand.want);
610
652
  }
@@ -614,6 +656,26 @@ async function runCommand(
614
656
  let pool: CpuPool | null = null;
615
657
  let reservation: Reservation | null = null;
616
658
  const releaseReservation = (): void => reservation?.release();
659
+ // A gated nextest leaf's case gate (case-gate.ts; specs/cowshed/05_gateway.md, "A case starts on a core"): served
660
+ // before the command starts, decided from the leaf's grant on, and closed once the command is gone, when every core a
661
+ // case still holds goes back to the pool and the leaf says what its cases ran on.
662
+ let caseGate: CaseGateServer | null = null;
663
+ let gated: GatedCases | null = null;
664
+ let caseGateClosed: Promise<void> | null = null;
665
+ const closeCaseGate = (): Promise<void> => {
666
+ caseGateClosed ??= (async () => {
667
+ const left = (await caseGate?.close()) ?? null;
668
+ if (left !== null) appendStderr(`bounded-exec: ${left}\n`);
669
+ const decided = gated;
670
+ if (decided === null) return;
671
+ decided.releaseAll();
672
+ const { own, lent, lentCoreMs, waitedCaseMs } = decided.cases.summary();
673
+ tell(
674
+ `cowshed: ${grant !== null ? 'cpu-pool' : 'mbx reserve'} cases own=${own} lent=${lent} lent_core_s=${Math.round(lentCoreMs / 1000)} waited_case_s=${Math.round(waitedCaseMs / 1000)}\n`,
675
+ );
676
+ })();
677
+ return caseGateClosed;
678
+ };
617
679
  // The run's progress channel (progress-channel.ts), served when the run has an idle bound for its records to restart,
618
680
  // and closed once its command is gone. A channel that leaves its socket behind says so; the run's verdict stands.
619
681
  let progressChannel: ProgressChannel | null = null;
@@ -647,6 +709,7 @@ async function runCommand(
647
709
  tell,
648
710
  step: (event) => stepTo(event),
649
711
  testing: () => isTesting(),
712
+ awakeNow,
650
713
  queueHost,
651
714
  });
652
715
  // Records the run's verdict when it has a task to record as. An `expiry` is set only when a bound ended the run.
@@ -655,7 +718,7 @@ async function runCommand(
655
718
  exitCode: number,
656
719
  expiry: ExpiredBound | null,
657
720
  elapsedMs: number,
658
- operational: boolean,
721
+ operational: OperationalCause | null,
659
722
  noSpace: NoSpaceSighting | null,
660
723
  lockWaits: readonly LockWait[],
661
724
  cpuRising: readonly CpuRise[],
@@ -684,9 +747,11 @@ async function runCommand(
684
747
  }
685
748
  });
686
749
  const admissionQueue = queue.queueWait(reservation !== null);
750
+ const cause: OperationalCause | null =
751
+ operational ?? (resultFailure === null ? null : { kind: 'executor', message: resultFailure });
687
752
  const verdict =
688
- operational || resultFailure !== null
689
- ? judgeOperationalFailure(exitCode, report)
753
+ cause !== null
754
+ ? judgeOperationalFailure(exitCode, report, cause)
690
755
  : judgeRun({
691
756
  exitCode,
692
757
  elapsedMs,
@@ -714,9 +779,10 @@ async function runCommand(
714
779
  appendStderr(message);
715
780
  await releaseCpu();
716
781
  await closeProgress();
782
+ await closeCaseGate();
717
783
  // The command never started, so its lease holds nothing of its own to keep.
718
784
  await releaseLease(false);
719
- await recordRun(1, null, Date.now() - asked, false, null, [], [], null);
785
+ await recordRun(1, null, Date.now() - asked, { kind: 'executor', message: message.trim() }, null, [], [], null);
720
786
  return { success: false, terminalOutput: outputChunks.join('') };
721
787
  };
722
788
  if (parentRefusal !== null) {
@@ -749,12 +815,12 @@ async function runCommand(
749
815
  if (idleTimeoutMs !== undefined) {
750
816
  const served = await serveProgressChannel({
751
817
  moved: () => {
752
- if (fixtureDone) stepTo({ kind: 'reported', at: Date.now(), testing: isTesting() });
818
+ if (fixtureDone) stepTo({ kind: 'reported', at: awakeNow(), testing: isTesting() });
753
819
  },
754
820
  invalid: (detail) => {
755
821
  if (lifecycleFailure !== null || state.expiry !== null || state.settled) return;
756
- lifecycleFailure = `progress channel: ${detail}`;
757
- appendStderr(`bounded-exec: ${lifecycleFailure}\n`);
822
+ lifecycleFailure = { kind: 'executor', message: `progress channel: ${detail}` };
823
+ appendStderr(`bounded-exec: ${lifecycleFailure.message}\n`);
758
824
  termination = terminate();
759
825
  },
760
826
  });
@@ -765,6 +831,35 @@ async function runCommand(
765
831
  }
766
832
  progressChannel = served.channel;
767
833
  }
834
+ // A case that arrives before the leaf's grant has no core to take and no leaf to wait for: its connection is refused,
835
+ // and its shim fails it by name.
836
+ if (reserved?.leaf.gated === true || lender !== null) {
837
+ const served = await serveCaseGate({
838
+ arrived: (id) => gated?.cases.arrived(id) ?? false,
839
+ ended: (id) => gated?.cases.ended(id),
840
+ invalid: (detail) => {
841
+ if (lifecycleFailure !== null || state.expiry !== null || state.settled) return;
842
+ lifecycleFailure = { kind: 'executor', message: `case gate: ${detail}` };
843
+ appendStderr(`bounded-exec: ${lifecycleFailure.message}\n`);
844
+ termination = terminate();
845
+ },
846
+ });
847
+ if (!served.ok) {
848
+ return refuse(
849
+ `bounded-exec: the leaf's case gate could not be served (${served.error}); refusing to run the command\n`,
850
+ );
851
+ }
852
+ caseGate = served.gate;
853
+ if (lender !== null) {
854
+ const gate = served.gate;
855
+ gated = gateCases(
856
+ lender.own,
857
+ lastWalls,
858
+ (id) => gate.start(id),
859
+ (id, left) => lendParentToken(lender.pool, `nextest:${id}`, left),
860
+ );
861
+ }
862
+ }
768
863
  const native = directory === null ? { command: sized.command, env: {} } : withTestResultReport(sized.command);
769
864
  const command =
770
865
  directory === null
@@ -783,11 +878,12 @@ async function runCommand(
783
878
  ...(await grantMemoryEnvironment(grant?.charged ?? null, process.env)),
784
879
  ...(leaf?.kind === 'cargo' ? { MBX_GRANT_FD: String(MBX_GRANT_FD) } : {}),
785
880
  ...(progressChannel === null ? {} : { [PROGRESS_SOCKET_ENV]: progressChannel.socket }),
881
+ ...(caseGate === null ? {} : { [TEST_GATE_ENV]: caseGate.socket }),
786
882
  };
787
883
  // A fixture rides as segment 0 of the lifecycle. Without the channel (Windows) it is plain shell text ahead
788
- // of the command: the progress bound runs during it and its own ceiling does not. Under mbx a leaf that names its
789
- // units counts them at the end of that segment, under the fixture's ceiling, its fixture's own output on stderr, and
790
- // reserves after it (`withCountedReservation`).
884
+ // of the command: the progress bound runs during it and its own ceiling does not. Under mbx a reserved leaf's
885
+ // fixture runs before its reservation, its ceiling from spawn (`withReservationAfterFixture`); a leaf that names its
886
+ // units counts them at the end of that segment, its fixture's own output on stderr (`withCountedReservation`).
791
887
  const counted =
792
888
  process.platform !== 'win32' && leaf?.kind === 'reserve' && fixture !== null && options.unitsCommand !== undefined;
793
889
  const segment =
@@ -796,15 +892,29 @@ async function runCommand(
796
892
  process.platform === 'win32' || segment === null ? null : withFixtureLifecycle(segment, command);
797
893
  const hooked = process.platform === 'win32' ? null : (fixtureHooked ?? withRunnerLifecycle(command));
798
894
  const lifecycle = hooked === null ? null : createRunnerLifecycle(hooked.runners);
799
- let lifecycleFailure: string | null = null;
895
+ let lifecycleFailure: OperationalCause | null = null;
896
+ /** Why the executor ended or failed the run whatever its command did, if it did: the verdict's operational cause. */
897
+ /** The cause the executor ended the command for, if it ended it (closures set `lifecycleFailure`). */
898
+ const endedFor = (): OperationalCause | null => lifecycleFailure;
899
+ const operationalCause = (): OperationalCause | null =>
900
+ endedFor() ??
901
+ (state.poolCloseError !== null
902
+ ? { kind: 'executor', message: `cpu pool did not close cleanly: ${state.poolCloseError}` }
903
+ : state.retireError !== null
904
+ ? { kind: 'executor', message: `could not retire the command's processes: ${state.retireError}` }
905
+ : null);
800
906
  const isTesting = (): boolean => lifecycleFailure !== null || (lifecycle?.isTesting() ?? true);
801
907
  const startedAt = Date.now();
802
908
  // With a supervisor, `cowshed subtree-root` asks it for a handle on itself, writes the answer to the handle
803
909
  // descriptor, and execs `/bin/sh -c command` in the same process: the command's pid, group and life are unchanged.
804
910
  const unadmitted = hooked?.command ?? (segment === null ? command : `${segment} && ${command}`);
805
911
  const shellCommand =
806
- leaf?.kind === 'reserve' && counted && fixtureHooked !== null
807
- ? withCountedReservation(fixtureHooked.fixture, fixtureHooked.main, leaf)
912
+ leaf?.kind === 'reserve' && fixtureHooked !== null
913
+ ? (counted ? withCountedReservation : withReservationAfterFixture)(
914
+ fixtureHooked.fixture,
915
+ fixtureHooked.main,
916
+ leaf,
917
+ )
808
918
  : withMbxLeaf(unadmitted, leaf);
809
919
  // Only leaves known to be scheduled can exclude zero-hold time. Others keep lifecycle hang protection.
810
920
  const queued = mbxIdleGated(leaf, sized.command, { ...process.env, ...env });
@@ -894,11 +1004,17 @@ async function runCommand(
894
1004
  command,
895
1005
  resultWriter === null
896
1006
  ? undefined
897
- : (result) => {
1007
+ : (closed) => {
1008
+ // nextest times a case from its shim's start: the wait at the gate is taken off its wall and named beside it.
1009
+ const waitMs = gated?.cases.waitedMs(closed.id) ?? 0;
1010
+ const result =
1011
+ closed.durationMs === null || waitMs === 0
1012
+ ? closed
1013
+ : { ...closed, durationMs: runWallMs(closed.durationMs, waitMs), waitMs };
898
1014
  const error = resultWriter?.append(result) ?? null;
899
1015
  if (error !== null && resultFailure === null) {
900
1016
  resultFailure = error;
901
- lifecycleFailure ??= error;
1017
+ lifecycleFailure ??= { kind: 'executor', message: error };
902
1018
  appendStderr(`bounded-exec: ${error}\n`);
903
1019
  termination = terminate();
904
1020
  }
@@ -918,56 +1034,57 @@ async function runCommand(
918
1034
  let fixtureDone = fixture === null || lifecycle === null;
919
1035
  // Whether a segment past the fixture ever announced: without one, the shell's exit is the fixture's.
920
1036
  let mainStarted = fixture === null || lifecycle === null;
921
- let captureTimer: NodeJS.Timeout | undefined;
922
- let idleCapture: { expiresAtMs: number; controller: AbortController; result: IdleDiagnosticResult } | undefined;
1037
+ let cancelCaptureTimer: (() => void) | undefined;
1038
+ let idleCapture:
1039
+ | { scheduledExpiryAtMs: number; controller: AbortController; result: IdleDiagnosticResult }
1040
+ | undefined;
923
1041
  const cancelCapture = (): void => {
924
- clearTimeout(captureTimer);
1042
+ cancelCaptureTimer?.();
925
1043
  idleCapture?.controller.abort();
926
1044
  };
927
- const scheduleCapture = (expiresAtMs: number): void => {
1045
+ const scheduleCapture = (awakeExpiresAtMs: number): void => {
928
1046
  const capture: NonNullable<typeof idleCapture> = {
929
- expiresAtMs,
1047
+ scheduledExpiryAtMs: Date.now() + Math.ceil(Math.max(0, awakeExpiresAtMs - awakeNow())),
930
1048
  controller: new AbortController(),
931
1049
  result: { kind: 'pending', startedAtMs: null },
932
1050
  };
933
1051
  idleCapture = capture;
934
- captureTimer = setTimeout(
935
- () => {
936
- if (state.settled || state.expiry !== null || idleCapture !== capture) return;
937
- const group = child.pid;
938
- if (group === undefined) {
939
- idleCapture.result = { kind: 'failed', detail: 'runner PID unavailable; no holder conclusion' };
940
- return;
941
- }
942
- idleCapture.result = { kind: 'pending', startedAtMs: Date.now() };
943
- void runnerPid
944
- .then((pid) => {
945
- if (pid === null)
946
- throw new Error('runner PID unavailable: its waiter named no command; no holder conclusion');
947
- return captureDiagnostics({
948
- pid,
949
- pgid: process.platform === 'win32' ? null : group,
950
- expiresAtMs: capture.expiresAtMs,
951
- deadlineMs: Math.max(0, Math.min(HANG_CAPTURE_MS, capture.expiresAtMs - Date.now())),
952
- signal: capture.controller.signal,
953
- });
954
- })
955
- .then(
956
- (snapshot) => {
957
- if (idleCapture === capture) idleCapture.result = { kind: 'captured', snapshot };
958
- },
959
- (error: unknown) => {
960
- if (idleCapture === capture) {
961
- idleCapture.result = {
962
- kind: 'failed',
963
- detail: error instanceof Error ? error.message : String(error),
964
- };
965
- }
966
- },
967
- );
968
- },
969
- Math.max(0, expiresAtMs - Date.now() - HANG_CAPTURE_MS),
970
- );
1052
+ cancelCaptureTimer = atClockDeadline(awakeExpiresAtMs - HANG_CAPTURE_MS, awakeNow, () => {
1053
+ if (state.settled || state.expiry !== null || idleCapture !== capture) return;
1054
+ const group = child.pid;
1055
+ if (group === undefined) {
1056
+ idleCapture.result = { kind: 'failed', detail: 'runner PID unavailable; no holder conclusion' };
1057
+ return;
1058
+ }
1059
+ idleCapture.result = { kind: 'pending', startedAtMs: Date.now() };
1060
+ capture.scheduledExpiryAtMs = Date.now() + Math.ceil(Math.max(0, awakeExpiresAtMs - awakeNow()));
1061
+ void runnerPid
1062
+ .then((pid) => {
1063
+ if (capture.controller.signal.aborted || awakeNow() >= awakeExpiresAtMs) return;
1064
+ if (pid === null)
1065
+ throw new Error('runner PID unavailable: its waiter named no command; no holder conclusion');
1066
+ return captureDiagnostics({
1067
+ pid,
1068
+ pgid: process.platform === 'win32' ? null : group,
1069
+ awakeExpiresAtMs,
1070
+ awakeNow,
1071
+ signal: capture.controller.signal,
1072
+ });
1073
+ })
1074
+ .then(
1075
+ (snapshot) => {
1076
+ if (snapshot !== undefined && idleCapture === capture) idleCapture.result = { kind: 'captured', snapshot };
1077
+ },
1078
+ (error: unknown) => {
1079
+ if (idleCapture === capture) {
1080
+ idleCapture.result = {
1081
+ kind: 'failed',
1082
+ detail: error instanceof Error ? error.message : String(error),
1083
+ };
1084
+ }
1085
+ },
1086
+ );
1087
+ });
971
1088
  };
972
1089
  const expiredAs = (kind: BoundKind): ExpiredBound =>
973
1090
  kind === 'idle'
@@ -976,25 +1093,27 @@ async function runCommand(
976
1093
  ? { kind, limitMs: fixture?.timeoutMs ?? 0 }
977
1094
  : { kind, limitMs: timeoutMs ?? 0 };
978
1095
  let checkingIdle = false;
1096
+ const nextBound = () => earliestBound(checkingIdle ? { ...bounds, idle: null } : bounds, awakeNow(), Date.now());
979
1097
  const armBoundTimer = (): void => {
980
1098
  clearTimeout(boundTimer);
981
1099
  if (state.settled || state.expiry !== null) return;
982
- const next = earliestBound(checkingIdle ? { ...bounds, idle: null } : bounds);
1100
+ const next = nextBound();
983
1101
  if (next !== null) {
984
- boundTimer = setTimeout(
985
- () => {
986
- void confirmExpiry(next.kind, next.at);
987
- },
988
- Math.max(0, next.at - Date.now()),
989
- );
1102
+ if (next.kind === 'idle' && idleCapture !== undefined && next.remainingMs > 0) {
1103
+ idleCapture.scheduledExpiryAtMs = Date.now() + Math.ceil(next.remainingMs);
1104
+ }
1105
+ boundTimer = setTimeout(() => void confirmExpiry(), Math.ceil(Math.max(0, next.remainingMs)));
990
1106
  }
991
1107
  };
992
- const confirmExpiry = async (kind: BoundKind, at: number): Promise<void> => {
1108
+ const confirmExpiry = async (): Promise<void> => {
993
1109
  if (state.settled || state.expiry !== null) return;
994
- const scheduled = earliestBound(checkingIdle ? { ...bounds, idle: null } : bounds);
995
- if (scheduled?.kind !== kind || scheduled.at !== at) return;
996
- if (kind === 'idle' && leaf?.kind === 'cargo') {
997
- if (checkingIdle) return;
1110
+ const scheduled = nextBound();
1111
+ if (scheduled === null) return;
1112
+ if (scheduled.remainingMs > 0) {
1113
+ armBoundTimer();
1114
+ return;
1115
+ }
1116
+ if (scheduled.kind === 'idle' && leaf?.kind === 'cargo') {
998
1117
  checkingIdle = true;
999
1118
  armBoundTimer();
1000
1119
  try {
@@ -1006,14 +1125,14 @@ async function runCommand(
1006
1125
  }
1007
1126
  }
1008
1127
  if (state.settled || state.expiry !== null) return;
1009
- const next = earliestBound(bounds);
1010
- if (next !== null && next.at <= Date.now()) expire(expiredAs(next.kind));
1128
+ const next = nextBound();
1129
+ if (next !== null && next.remainingMs <= 0) expire(expiredAs(next.kind));
1011
1130
  else armBoundTimer();
1012
1131
  };
1013
1132
  const stepTo = (event: BoundEvent): void => {
1014
1133
  if (state.settled || state.expiry !== null) return;
1015
1134
  const idleBefore = bounds.idle;
1016
- bounds = stepBounds(
1135
+ const next = stepBounds(
1017
1136
  bounds,
1018
1137
  {
1019
1138
  ...(idleTimeoutMs === undefined ? {} : { idleMs: idleTimeoutMs }),
@@ -1022,6 +1141,8 @@ async function runCommand(
1022
1141
  },
1023
1142
  event,
1024
1143
  );
1144
+ if (next === bounds) return;
1145
+ bounds = next;
1025
1146
  if (bounds.idle !== idleBefore) {
1026
1147
  cancelCapture();
1027
1148
  idleCapture = undefined;
@@ -1034,14 +1155,14 @@ async function runCommand(
1034
1155
  // through appendStderr as well, and re-arming on those would let a teardown
1035
1156
  // message extend the very bound that just fired.
1036
1157
  const updateIdle = (finished: boolean, wasTesting: boolean): void => {
1037
- stepTo({ kind: 'progress', at: Date.now(), testing: isTesting(), wasTesting, finished });
1158
+ stepTo({ kind: 'progress', at: awakeNow(), testing: isTesting(), wasTesting, finished });
1038
1159
  };
1039
1160
  const scanProgress = (stream: 'stdout' | 'stderr', text: Buffer | string): void => {
1040
1161
  if (!fixtureDone) return;
1041
1162
  updateIdle(progress.scan(stream, text), isTesting());
1042
1163
  const error = progress.error();
1043
1164
  if (error !== null && lifecycleFailure === null) {
1044
- lifecycleFailure = error;
1165
+ lifecycleFailure = { kind: 'executor', message: error };
1045
1166
  appendStderr(`bounded-exec: ${error}\n`);
1046
1167
  termination = terminate();
1047
1168
  }
@@ -1051,7 +1172,7 @@ async function runCommand(
1051
1172
  if (!(channel instanceof Duplex)) throw new Error('runner lifecycle requires a duplex child descriptor');
1052
1173
  const failLifecycle = (detail: string, wasTesting = isTesting()): void => {
1053
1174
  if (lifecycleFailure !== null || state.expiry !== null) return;
1054
- lifecycleFailure = detail;
1175
+ lifecycleFailure = { kind: 'executor', message: `runner lifecycle failed: ${detail}` };
1055
1176
  appendStderr(`bounded-exec: runner lifecycle failed: ${detail}\n`);
1056
1177
  updateIdle(false, wasTesting);
1057
1178
  };
@@ -1127,6 +1248,7 @@ async function runCommand(
1127
1248
  return killChildTree();
1128
1249
  })
1129
1250
  .finally(closeProgress)
1251
+ .finally(closeCaseGate)
1130
1252
  .finally(releaseCpu)
1131
1253
  .finally(() => releaseLease(true))
1132
1254
  .finally(releaseReservation)
@@ -1135,7 +1257,7 @@ async function runCommand(
1135
1257
  signalToExitCode(signal),
1136
1258
  state.expiry,
1137
1259
  Date.now() - startedAt,
1138
- state.poolCloseError !== null || lifecycleFailure !== null || state.retireError !== null,
1260
+ operationalCause(),
1139
1261
  state.noSpace === null ? null : await state.noSpace,
1140
1262
  state.lockWaits,
1141
1263
  state.cpuRising,
@@ -1211,7 +1333,7 @@ async function runCommand(
1211
1333
  appendStderr(
1212
1334
  `Hang diagnostics: runner pid=${runner ?? 'unavailable'} pgid=${
1213
1335
  process.platform === 'win32' ? 'unavailable' : (child.pid ?? 'unavailable')
1214
- } expiryAtMs=${Date.now()} scheduledExpiryAtMs=${idleCapture?.expiresAtMs ?? 'unavailable'}\n`,
1336
+ } expiryAtMs=${Date.now()} scheduledExpiryAtMs=${idleCapture?.scheduledExpiryAtMs ?? 'unavailable'}\n`,
1215
1337
  );
1216
1338
  if (observed?.kind === 'captured') {
1217
1339
  appendStderr(
@@ -1299,22 +1421,27 @@ async function runCommand(
1299
1421
  // fire on a command the fixture already handed over to.
1300
1422
  stepTo({
1301
1423
  kind: 'spawn',
1302
- at: Date.now(),
1424
+ at: awakeNow(),
1425
+ wallAt: Date.now(),
1303
1426
  testing: isTesting(),
1304
1427
  fixture: fixture !== null && lifecycle !== null,
1305
- fixtureFirst: counted && lifecycle !== null,
1428
+ fixtureFirst: leaf?.kind === 'reserve' && fixtureHooked !== null,
1306
1429
  queued,
1307
1430
  });
1308
1431
  if (leaf !== null) {
1309
1432
  const channel = child.stdio.at(MBX_GRANT_FD);
1310
1433
  if (!(channel instanceof Duplex)) throw new Error('an mbx leaf requires its mbx descriptor');
1311
- const failGrant = (detail: string): void => {
1434
+ const descriptorFailure = (detail: string): OperationalCause => ({
1435
+ kind: 'executor',
1436
+ message: `mbx grant descriptor: ${detail}`,
1437
+ });
1438
+ const failGrant = (cause: OperationalCause): void => {
1312
1439
  if (lifecycleFailure !== null || state.expiry !== null || state.settled) return;
1313
- lifecycleFailure ??= `mbx grant descriptor: ${detail}`;
1314
- appendStderr(`bounded-exec: ${lifecycleFailure}\n`);
1440
+ lifecycleFailure = cause;
1441
+ appendStderr(`bounded-exec: ${cause.message}\n`);
1315
1442
  termination = terminate();
1316
1443
  };
1317
- channel.on('error', (error) => failGrant(error.message));
1444
+ channel.on('error', (error) => failGrant(descriptorFailure(error.message)));
1318
1445
  if (reserved !== null) {
1319
1446
  // The shell asks for its cores in one line and reads the count granted (mbx-pool.ts, `reservation`). A refusal
1320
1447
  // ends the descriptor unanswered: the shell's read fails, and nothing runs.
@@ -1323,7 +1450,7 @@ async function runCommand(
1323
1450
  channel.setEncoding('utf8');
1324
1451
  channel.on('data', (text: string) => {
1325
1452
  if (reservation !== null) {
1326
- failGrant(`the command asked for cores a second time: ${JSON.stringify(text)}`);
1453
+ failGrant(descriptorFailure(`the command asked for cores a second time: ${JSON.stringify(text)}`));
1327
1454
  return;
1328
1455
  }
1329
1456
  asked += text;
@@ -1332,17 +1459,30 @@ async function runCommand(
1332
1459
  const line = asked.slice(0, end);
1333
1460
  const cpus = askedCores(line, reservedLeaf.demand.want);
1334
1461
  if (cpus === null) {
1335
- failGrant(`the command asked for ${JSON.stringify(line)} cores, not 1 to ${reservedLeaf.demand.want}`);
1462
+ failGrant(
1463
+ descriptorFailure(
1464
+ `the command asked for ${JSON.stringify(line)} cores, not 1 to ${reservedLeaf.demand.want}`,
1465
+ ),
1466
+ );
1336
1467
  channel.end();
1337
1468
  return;
1338
1469
  }
1339
1470
  queue.waiting();
1340
- tell(`cowshed: mbx reserve wait start cpus=${cpus} priority=${reservedLeaf.priority}\n`);
1341
- const held = reserveCores(endpoint, cpus, reservedLeaf.priority);
1471
+ // A whole-suite test leaf names the fewest threads at which its recorded walls keep a test finishing within
1472
+ // its own no-progress bound (specs/cowshed/05_gateway.md, "A minimum for a whole suite").
1473
+ const min = idleTimeoutMs === undefined ? 1 : suiteMinimum([...lastWalls.values()], cpus, idleTimeoutMs);
1474
+ const named = min > 1 ? ` min=${min}` : '';
1475
+ tell(`cowshed: mbx reserve wait start cpus=${cpus}${named} priority=${reservedLeaf.priority}\n`);
1476
+ const held = reserveCores(endpoint, cpus, reservedLeaf.priority, min);
1342
1477
  reservation = held;
1343
1478
  void held.granted.then((granted) => {
1344
1479
  if (!granted.ok) {
1345
- failGrant(`mbx did not grant the reservation: ${granted.error}`);
1480
+ failGrant({
1481
+ kind: 'reservation-refused',
1482
+ endpoint: granted.refusal.endpoint,
1483
+ errno: granted.refusal.errno,
1484
+ message: `mbx did not grant the reservation: ${granted.refusal.message}`,
1485
+ });
1346
1486
  channel.end();
1347
1487
  return;
1348
1488
  }
@@ -1350,8 +1490,20 @@ async function runCommand(
1350
1490
  if (state.settled || state.expiry !== null) return;
1351
1491
  queue.admitted = true;
1352
1492
  tell(`cowshed: mbx reserve granted elapsed=${Date.now() - startedAt}ms priority=${reservedLeaf.priority}\n`);
1493
+ if (caseGate !== null) {
1494
+ const gate = caseGate;
1495
+ gated = gateCases(
1496
+ granted.cpus,
1497
+ lastWalls,
1498
+ (id) => gate.start(id),
1499
+ (id) => {
1500
+ const held = reserveCores(endpoint, 1, 'low', 1, `nextest:${id}`);
1501
+ return { granted: held.granted.then((answer) => answer.ok), release: () => held.release() };
1502
+ },
1503
+ );
1504
+ }
1353
1505
  channel.end(`${granted.cpus}\n`);
1354
- stepTo({ kind: 'granted', at: Date.now(), testing: isTesting(), fixture: !fixtureDone });
1506
+ stepTo({ kind: 'granted', at: awakeNow(), wallAt: Date.now(), testing: isTesting(), fixture: !fixtureDone });
1355
1507
  void granted.lost.then((why) => {
1356
1508
  if (state.settled || cpuReleased !== null) return;
1357
1509
  appendStderr(
@@ -1371,11 +1523,17 @@ async function runCommand(
1371
1523
  said = true;
1372
1524
  tell(`cowshed: mbx cargo granted elapsed=${Date.now() - startedAt}ms\n`);
1373
1525
  }
1374
- stepTo({ kind: 'granted', at: Date.now(), testing: isTesting(), fixture: !fixtureDone });
1526
+ stepTo({
1527
+ kind: 'granted',
1528
+ at: awakeNow(),
1529
+ wallAt: Date.now(),
1530
+ testing: isTesting(),
1531
+ fixture: !fixtureDone,
1532
+ });
1375
1533
  } else if (byte === 114) {
1376
- stepTo({ kind: 'released', at: Date.now() });
1534
+ stepTo({ kind: 'released', at: awakeNow() });
1377
1535
  } else {
1378
- failGrant(`unexpected byte ${byte}`);
1536
+ failGrant(descriptorFailure(`unexpected byte ${byte}`));
1379
1537
  return;
1380
1538
  }
1381
1539
  }
@@ -1415,16 +1573,22 @@ async function runCommand(
1415
1573
  // The runner is gone, and with it every test that could report on the run's progress channel. Its sub-pool closes
1416
1574
  // first, so no descendant still draws on tokens the grant returns; then the grant goes back to its parent or the host.
1417
1575
  await closeProgress();
1576
+ await closeCaseGate();
1418
1577
  await releaseCpu();
1419
1578
 
1420
1579
  const code = exitCode ?? signalToExitCode(exitSignal);
1421
1580
  if (code !== 0 && state.expiry === null) {
1422
- // No bound of this executor ended it, so a signal it died of came from outside (the kernel's OOM killer sends
1423
- // SIGKILL): the signal is named, never folded into a bare status.
1581
+ // The executor ended it for a cause of its own, which is said; otherwise no bound of this executor ended it, so a
1582
+ // signal it died of came from outside (the kernel's OOM killer sends SIGKILL): the signal is named, never folded
1583
+ // into a bare status.
1584
+ const signalled = exitCode === null && exitSignal !== null;
1585
+ const endedBy = endedFor();
1424
1586
  const ended =
1425
- exitCode === null && exitSignal !== null
1426
- ? `was killed by ${exitSignal} (status ${code})`
1427
- : `exited with status ${code}`;
1587
+ endedBy !== null
1588
+ ? `was ended by bounded-exec because ${endedBy.message} (${signalled ? `${exitSignal}, ` : ''}status ${code})`
1589
+ : signalled
1590
+ ? `was killed by ${exitSignal} (status ${code})`
1591
+ : `exited with status ${code}`;
1428
1592
  appendStderr(
1429
1593
  fixture !== null && !mainStarted
1430
1594
  ? `Fixture command ${ended}: ${segment ?? fixture.command}\n`
@@ -1459,9 +1623,19 @@ async function runCommand(
1459
1623
  }
1460
1624
  const peakBytes = reported?.kind === 'measured' ? reported.bytes : null;
1461
1625
 
1626
+ // A Go run sums nothing itself, and a reader that keeps part of a long output cannot count it: the sum ends the output.
1627
+ const goRun = progress.goSummary();
1628
+ if (goRun !== null) {
1629
+ appendStdout(
1630
+ goRunSummaryText(
1631
+ goRun,
1632
+ declaredGoTests(goRun.failedTests, cwd, context.root, mergeEnv(lease?.directory ?? ordinaryTemp, env)),
1633
+ ),
1634
+ );
1635
+ }
1462
1636
  // Broken resource ownership, a lost native lifecycle, or leftovers that could not be retired fail even if a shell
1463
1637
  // branch later exited 0: its verdict is an ordinary operational failure, never a successful or silently unbounded run.
1464
- const operationalFailure = state.poolCloseError !== null || lifecycleFailure !== null || state.retireError !== null;
1638
+ const operationalFailure = operationalCause();
1465
1639
  await recordRun(
1466
1640
  code,
1467
1641
  state.expiry,
@@ -1479,7 +1653,7 @@ async function runCommand(
1479
1653
  code,
1480
1654
  });
1481
1655
  return {
1482
- success: !operationalFailure && resultFailure === null && state.expiry === null && code === 0,
1656
+ success: operationalFailure === null && resultFailure === null && state.expiry === null && code === 0,
1483
1657
  terminalOutput: outputChunks.join(''),
1484
1658
  ...(killedForMemory ? { killedForMemory: true } : {}),
1485
1659
  };