@smoothbricks/nx-plugin 0.6.2 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (177) hide show
  1. package/README.md +417 -171
  2. package/dist/.build-inputs.json +39 -28
  3. package/dist/bin/smoo-nx-go-test-build.d.ts +3 -0
  4. package/dist/bin/smoo-nx-go-test-build.d.ts.map +1 -0
  5. package/dist/bin/smoo-nx-go-test-build.js +79 -0
  6. package/dist/bin/smoo-nx-go-test-run.d.ts +3 -0
  7. package/dist/bin/smoo-nx-go-test-run.d.ts.map +1 -0
  8. package/dist/bin/smoo-nx-go-test-run.js +129 -0
  9. package/dist/bin/smoo-nx-publish.d.ts +3 -0
  10. package/dist/bin/smoo-nx-publish.d.ts.map +1 -0
  11. package/dist/bin/smoo-nx-publish.js +3 -0
  12. package/dist/bounded-test-policy.d.ts +1 -1
  13. package/dist/bounded-test-policy.d.ts.map +1 -1
  14. package/dist/bounded-test-policy.js +9 -8
  15. package/dist/cargo-retention-hook.d.ts +6 -2
  16. package/dist/cargo-retention-hook.d.ts.map +1 -1
  17. package/dist/cargo-retention-hook.js +13 -7
  18. package/dist/cargo-workspace.d.ts +10 -20
  19. package/dist/cargo-workspace.d.ts.map +1 -1
  20. package/dist/cargo-workspace.js +12 -22
  21. package/dist/cross-check-policy.d.ts +2 -2
  22. package/dist/cross-check-policy.js +2 -2
  23. package/dist/deploy-policy.d.ts +6 -0
  24. package/dist/deploy-policy.d.ts.map +1 -1
  25. package/dist/deploy-policy.js +11 -0
  26. package/dist/eslint-file-set.d.ts +4 -1
  27. package/dist/eslint-file-set.d.ts.map +1 -1
  28. package/dist/eslint-sources.d.ts +25 -0
  29. package/dist/eslint-sources.d.ts.map +1 -0
  30. package/dist/eslint-sources.js +99 -0
  31. package/dist/executors/all-verdicts/executor.d.ts +2 -1
  32. package/dist/executors/all-verdicts/executor.d.ts.map +1 -1
  33. package/dist/executors/all-verdicts/executor.js +48 -2
  34. package/dist/executors/all-verdicts/schema.d.ts +13 -0
  35. package/dist/executors/all-verdicts/schema.d.ts.map +1 -1
  36. package/dist/executors/all-verdicts/schema.js +7 -0
  37. package/dist/executors/bounded-exec/bounds.d.ts +57 -8
  38. package/dist/executors/bounded-exec/bounds.d.ts.map +1 -1
  39. package/dist/executors/bounded-exec/bounds.js +100 -11
  40. package/dist/executors/bounded-exec/executor.d.ts +2 -1
  41. package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
  42. package/dist/executors/bounded-exec/executor.js +289 -34
  43. package/dist/executors/bounded-exec/hang-diagnostics.d.ts +2 -0
  44. package/dist/executors/bounded-exec/hang-diagnostics.d.ts.map +1 -1
  45. package/dist/executors/bounded-exec/hang-diagnostics.js +2 -2
  46. package/dist/executors/bounded-exec/mbx-pool.d.ts +56 -10
  47. package/dist/executors/bounded-exec/mbx-pool.d.ts.map +1 -1
  48. package/dist/executors/bounded-exec/mbx-pool.js +145 -11
  49. package/dist/executors/bounded-exec/mbx-queue.d.ts +30 -0
  50. package/dist/executors/bounded-exec/mbx-queue.d.ts.map +1 -0
  51. package/dist/executors/bounded-exec/mbx-queue.js +229 -0
  52. package/dist/executors/bounded-exec/progress.d.ts +9 -2
  53. package/dist/executors/bounded-exec/progress.d.ts.map +1 -1
  54. package/dist/executors/bounded-exec/progress.js +151 -25
  55. package/dist/executors/bounded-exec/runner-sizing.d.ts +40 -2
  56. package/dist/executors/bounded-exec/runner-sizing.d.ts.map +1 -1
  57. package/dist/executors/bounded-exec/runner-sizing.js +55 -4
  58. package/dist/executors/bounded-exec/runtime-pin.js +1 -1
  59. package/dist/executors/bounded-exec/test-result-writer.d.ts +30 -0
  60. package/dist/executors/bounded-exec/test-result-writer.d.ts.map +1 -0
  61. package/dist/executors/bounded-exec/test-result-writer.js +126 -0
  62. package/dist/executors/bounded-exec/test-results.d.ts +33 -0
  63. package/dist/executors/bounded-exec/test-results.d.ts.map +1 -0
  64. package/dist/executors/bounded-exec/test-results.js +33 -0
  65. package/dist/executors/bounded-exec/verdict.d.ts +26 -2
  66. package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
  67. package/dist/executors/bounded-exec/verdict.js +108 -4
  68. package/dist/executors/wrangler-deploy/deploy-hash.d.ts +10 -0
  69. package/dist/executors/wrangler-deploy/deploy-hash.d.ts.map +1 -0
  70. package/dist/executors/wrangler-deploy/deploy-hash.js +14 -0
  71. package/dist/executors/wrangler-deploy/executor.d.ts +24 -0
  72. package/dist/executors/wrangler-deploy/executor.d.ts.map +1 -0
  73. package/dist/executors/wrangler-deploy/executor.js +43 -0
  74. package/dist/executors/wrangler-deploy/hasher.d.ts +27 -0
  75. package/dist/executors/wrangler-deploy/hasher.d.ts.map +1 -0
  76. package/dist/executors/wrangler-deploy/hasher.js +90 -0
  77. package/dist/executors/wrangler-deploy/schema.d.ts +5 -0
  78. package/dist/executors/wrangler-deploy/schema.d.ts.map +1 -0
  79. package/dist/go-tests.d.ts +30 -0
  80. package/dist/go-tests.d.ts.map +1 -0
  81. package/dist/go-tests.js +43 -0
  82. package/dist/index.d.ts +12 -4
  83. package/dist/index.d.ts.map +1 -1
  84. package/dist/index.js +338 -295
  85. package/dist/managed-files/files.d.ts.map +1 -1
  86. package/dist/managed-files/files.js +5 -0
  87. package/dist/nx-publish.d.ts +24 -0
  88. package/dist/nx-publish.d.ts.map +1 -0
  89. package/dist/nx-publish.js +150 -0
  90. package/dist/process-group.js +2 -2
  91. package/dist/runtime-input-policy.d.ts +17 -3
  92. package/dist/runtime-input-policy.d.ts.map +1 -1
  93. package/dist/runtime-input-policy.js +4 -4
  94. package/dist/test-timings.d.ts +23 -4
  95. package/dist/test-timings.d.ts.map +1 -1
  96. package/dist/test-timings.js +72 -12
  97. package/dist/tool-bins.d.ts +6 -8
  98. package/dist/tool-bins.d.ts.map +1 -1
  99. package/dist/tool-bins.js +7 -5
  100. package/dist/version-actions.cjs +10 -11
  101. package/dist/version-actions.d.cts.map +1 -1
  102. package/dist/workspace-config-policy.d.ts.map +1 -1
  103. package/dist/workspace-config-policy.js +17 -22
  104. package/dist/workspace-index.d.ts +7 -0
  105. package/dist/workspace-index.d.ts.map +1 -1
  106. package/dist/workspace-index.js +11 -0
  107. package/dist/workspace-manifest.js +3 -3
  108. package/executors.json +7 -6
  109. package/managed/raw/patches/nx@23.2.1.patch +118 -24
  110. package/managed/raw/patches/ttsc@0.34.1.patch +139 -0
  111. package/managed/raw/tooling/devenv +1 -2
  112. package/managed/raw/tooling/direnv/developer-links.ts +124 -51
  113. package/managed/raw/tooling/direnv/devenv.smoo.nix +34 -15
  114. package/managed/raw/tooling/direnv/github-actions-bootstrap.sh +20 -10
  115. package/managed/raw/tooling/direnv/installed-bun-links.ts +7 -6
  116. package/managed/raw/tooling/direnv/setup-environment.ts +49 -24
  117. package/managed/raw/tooling/direnv/ttsc-env.sh +8 -6
  118. package/managed/raw/tooling/git-hooks/pinned-revisions-served.ts +252 -0
  119. package/managed/raw/tooling/git-hooks/pre-push.sh +34 -32
  120. package/managed/templates/github/actions/cache-node-modules/action.yml +1 -1
  121. package/managed/templates/github/actions/cache-ttsc-plugins/action.yml +12 -11
  122. package/package.json +277 -11
  123. package/runtime/nx-publish/.build-inputs.json +11 -0
  124. package/runtime/nx-publish/cache_adapter.js +45 -0
  125. package/runtime/nx-publish/nx_publish.js +60 -0
  126. package/runtime-input.sh +62 -26
  127. package/src/bin/smoo-nx-go-test-build.ts +83 -0
  128. package/src/bin/smoo-nx-go-test-run.ts +140 -0
  129. package/src/bin/smoo-nx-publish.ts +4 -0
  130. package/src/bounded-test-policy.ts +9 -8
  131. package/src/cargo-retention-hook.ts +16 -10
  132. package/src/cargo-workspace.ts +12 -24
  133. package/src/cross-check-policy.ts +2 -2
  134. package/src/deploy-policy.ts +11 -0
  135. package/src/eslint-file-set.ts +4 -1
  136. package/src/eslint-sources.ts +134 -0
  137. package/src/executors/all-verdicts/executor.ts +58 -3
  138. package/src/executors/all-verdicts/schema.json +4 -0
  139. package/src/executors/all-verdicts/schema.ts +14 -0
  140. package/src/executors/bounded-exec/bounds.ts +132 -15
  141. package/src/executors/bounded-exec/executor.ts +329 -34
  142. package/src/executors/bounded-exec/hang-diagnostics.ts +3 -2
  143. package/src/executors/bounded-exec/mbx-pool.ts +158 -17
  144. package/src/executors/bounded-exec/mbx-queue.ts +170 -0
  145. package/src/executors/bounded-exec/progress.ts +171 -18
  146. package/src/executors/bounded-exec/runner-sizing.ts +73 -7
  147. package/src/executors/bounded-exec/runtime-pin.ts +1 -1
  148. package/src/executors/bounded-exec/test-result-writer.ts +152 -0
  149. package/src/executors/bounded-exec/test-results.ts +39 -0
  150. package/src/executors/bounded-exec/verdict.ts +115 -5
  151. package/src/executors/wrangler-deploy/deploy-hash.ts +15 -0
  152. package/src/executors/wrangler-deploy/executor.ts +63 -0
  153. package/src/executors/wrangler-deploy/hasher.ts +111 -0
  154. package/src/executors/wrangler-deploy/schema.json +16 -0
  155. package/src/executors/wrangler-deploy/schema.ts +4 -0
  156. package/src/go-tests.ts +67 -0
  157. package/src/index.ts +368 -323
  158. package/src/managed-files/files.ts +5 -0
  159. package/src/nx-publish.ts +157 -0
  160. package/src/process-group.ts +2 -2
  161. package/src/runtime-input-policy.ts +4 -4
  162. package/src/test-timings.ts +72 -14
  163. package/src/tool-bins.ts +13 -9
  164. package/src/version-actions.cts +10 -11
  165. package/src/workspace-config-policy.ts +20 -29
  166. package/src/workspace-index.ts +12 -0
  167. package/src/workspace-manifest.ts +3 -3
  168. package/test-timings.json +131 -125
  169. package/dist/executors/clean-outputs/executor.d.ts +0 -26
  170. package/dist/executors/clean-outputs/executor.d.ts.map +0 -1
  171. package/dist/executors/clean-outputs/executor.js +0 -83
  172. package/dist/executors/clean-outputs/schema.d.ts +0 -4
  173. package/dist/executors/clean-outputs/schema.d.ts.map +0 -1
  174. package/src/executors/clean-outputs/executor.ts +0 -131
  175. package/src/executors/clean-outputs/schema.json +0 -15
  176. package/src/executors/clean-outputs/schema.ts +0 -3
  177. /package/dist/executors/{clean-outputs → wrangler-deploy}/schema.js +0 -0
@@ -1,4 +1,4 @@
1
- import { type ChildProcess, spawn } from 'node:child_process';
1
+ import { type ChildProcess, type IOType, spawn } from 'node:child_process';
2
2
  import { accessSync, constants } from 'node:fs';
3
3
  import { mkdtemp, rm } from 'node:fs/promises';
4
4
  import { availableParallelism, userInfo } from 'node:os';
@@ -13,7 +13,17 @@ import {
13
13
  processGroupHasMembers,
14
14
  signalProcessGroup,
15
15
  } from '../../process-group.js';
16
- import { type BoundEvent, type BoundKind, type Bounds, earliestBound, stepBounds, UNARMED } from './bounds.js';
16
+ import {
17
+ admissionWaitMs,
18
+ type BoundEvent,
19
+ type BoundKind,
20
+ type Bounds,
21
+ earliestBound,
22
+ NO_ADMISSION_WAIT,
23
+ stepAdmissionWait,
24
+ stepBounds,
25
+ UNARMED,
26
+ } from './bounds.js';
17
27
  import {
18
28
  CPU_POOL_ENV,
19
29
  type CpuPool,
@@ -28,7 +38,16 @@ import type { HangDiagnostics } from './hang-diagnostics.js';
28
38
  import { captureHangDiagnostics, HANG_CAPTURE_MS } from './hang-diagnostics.js';
29
39
  import type { CpuWaitRow } from './lease-wait.js';
30
40
  import type { LockWait } from './lock-wait.js';
31
- import { mbxLeaf, mbxPool, withMbxLeaf } from './mbx-pool.js';
41
+ import {
42
+ MBX_GRANT_FD,
43
+ mbxIdleGated,
44
+ mbxLeaf,
45
+ mbxPool,
46
+ nxTaskCores,
47
+ runsPrebuiltNextest,
48
+ withMbxLeaf,
49
+ } from './mbx-pool.js';
50
+ import { MBX_QUEUE_POLL_MS, type MbxQueuedRequest, snapshotMbxQueue } from './mbx-queue.js';
32
51
  import { createNoSpaceWatch } from './no-space.js';
33
52
  import {
34
53
  COWSHED_EXECUTABLE_ENV,
@@ -39,7 +58,7 @@ import {
39
58
  type SubtreeAdmission,
40
59
  type SubtreeOwner,
41
60
  } from './owned-subtree.js';
42
- import { createRunProgress, withProgressReport } from './progress.js';
61
+ import { createRunProgress, withProgressReport, withTestResultReport } from './progress.js';
43
62
  import {
44
63
  createHostCommands,
45
64
  describeCoverageGap,
@@ -53,13 +72,17 @@ import {
53
72
  ramTempPaths,
54
73
  } from './ram-temp.js';
55
74
  import { createRunnerLifecycle, withFixtureLifecycle, withRunnerLifecycle } from './runner-lifecycle.js';
56
- import { CPU_TOKENS_ENV, runnerDemand, sizeRunner } from './runner-sizing.js';
75
+ import { CPU_TOKENS_ENV, runnerDemand, sizeRunner, sizeRunnerToGrant } from './runner-sizing.js';
57
76
  import { checkRuntimePin, describeRuntimePinRefusal } from './runtime-pin.js';
58
77
  import type { BoundedExecOptions } from './schema.js';
78
+ import { createTestResultWriter, type TestResultWriter } from './test-result-writer.js';
59
79
  import {
80
+ type AdmissionQueueWait,
60
81
  type BoundedExecRecord,
61
82
  type BoundedExecTask,
83
+ BUN_TEST,
62
84
  clearTaskDirectory,
85
+ describeAdmissionQueue,
63
86
  describeNoSpace,
64
87
  type ExpiredBound,
65
88
  JUNIT_FILE,
@@ -291,6 +314,7 @@ export async function runBoundedExec(
291
314
  inherited: InheritedCpuPool | null = null,
292
315
  captureDiagnostics: typeof captureHangDiagnostics = captureHangDiagnostics,
293
316
  owner: SubtreeOwner | null = null,
317
+ queueHost?: Parameters<typeof snapshotMbxQueue>[3],
294
318
  ): Promise<BoundedExecResult> {
295
319
  const cwd = resolveCwd(options.cwd, context.root);
296
320
  const directory = context.record?.directory ?? null;
@@ -307,6 +331,8 @@ export async function runBoundedExec(
307
331
  ? { command: options.fixtureCommand, timeoutMs: options.fixtureTimeoutMs }
308
332
  : null;
309
333
  const outputChunks: string[] = [];
334
+ let resultWriter: TestResultWriter | null = null;
335
+ let resultFailure: string | null = null;
310
336
  const state: RunState = {
311
337
  settled: false,
312
338
  expiry: null,
@@ -511,7 +537,12 @@ export async function runBoundedExec(
511
537
  // (`run-cpu-pool.ts`), so a task takes its tokens from a parent's pool; a command that inherits none runs exactly as
512
538
  // asked and nothing is said.
513
539
  const cores = availableParallelism();
514
- const demand = runnerDemand(requested, options.parallelism, cores, process.env, options.env);
540
+ // A prebuilt nextest leaf under mbx asks for its task's share of the host (`nxTaskCores`), not every core: it knows
541
+ // no count of its tests. A declared or named count still wins.
542
+ const mbx = mbxPool(process.env);
543
+ const leafCores =
544
+ mbx !== null && runsPrebuiltNextest(requested) ? nxTaskCores(cores, { ...process.env, ...options.env }) : cores;
545
+ const demand = runnerDemand(requested, options.parallelism, leafCores, process.env, options.env);
515
546
  const asked = Date.now();
516
547
  // A pool that answered and refused is a live budget, and running beside it would count the run twice: the run fails
517
548
  // instead. A pool whose endpoint is gone (its holder exited) lends nothing, and the run goes on as asked.
@@ -532,17 +563,18 @@ export async function runBoundedExec(
532
563
  const waitedMs = Date.now() - asked;
533
564
  let cpu: BoundedExecRecord['cpu'];
534
565
  let sized: { command: string; env: Record<string, string> } = { command: requested, env: {} };
535
- // With mbx the leaf's admission is mbx's one pool, not a parent's tokens: its runner is sized to the job's count,
536
- // and a non-Cargo leaf holds that much of the pool for its run (specs/cowshed/05_gateway.md, "Parallelism inside a
537
- // job"). No grant and no sub-pool beside it.
538
- const mbx = mbxPool(process.env);
539
- const leaf = mbx === null ? null : mbxLeaf(mbx, demand.kind, demand.want);
566
+ // With mbx the leaf's admission is mbx's one pool, not a parent's tokens. A leaf mbx does not admit itself asks for
567
+ // its runner's count, a maximum, and its runner runs on the cores it is granted, which only the command learns
568
+ // (specs/cowshed/05_gateway.md, "Parallelism inside a job"). No grant and no sub-pool beside it.
569
+ const leaf = mbx === null ? null : mbxLeaf(mbx, demand, requested);
540
570
  if (grant !== null) {
541
571
  sized = sizeRunner(demand, requested, grant.tokens);
542
572
  cpu = { tokens: grant.tokens, want: demand.want, waitedMs };
543
573
  appendStderr(
544
574
  `cowshed: cpu-pool wait done elapsed=${waitedMs}ms status=ok tokens=${grant.tokens}/${demand.want} runner=${demand.kind}\n`,
545
575
  );
576
+ } else if (leaf?.kind === 'reserve') {
577
+ sized = { command: sizeRunnerToGrant(leaf.demand, requested), env: {} };
546
578
  } else if (leaf !== null) {
547
579
  sized = sizeRunner(demand, requested, demand.want);
548
580
  }
@@ -564,9 +596,34 @@ export async function runBoundedExec(
564
596
  })();
565
597
  return cpuReleased;
566
598
  };
599
+ let mbxAdmitted = false;
600
+ let admissionWait = NO_ADMISSION_WAIT;
601
+ let lastQueueRequests: readonly MbxQueuedRequest[] = [];
602
+ /**
603
+ * Why the last admission snapshot failed, while the one after it has not succeeded; null otherwise. A snapshot only
604
+ * decides whether waiting on mbx admission is credited against the idle bound, so a failed one withholds that credit
605
+ * and ends nothing: the bounds decide, and an idle expiry names it.
606
+ */
607
+ let unconfirmedAdmission: string | null = null;
608
+ /** When the last successful snapshot completed (`Date.now()`): credit runs to it and no further. */
609
+ let confirmedAdmissionAt: number | null = null;
610
+ const queueWait = (): AdmissionQueueWait | null =>
611
+ mbxAdmitted || lastQueueRequests.length === 0
612
+ ? null
613
+ : {
614
+ waitedMs: admissionWaitMs(admissionWait, performance.now()),
615
+ requests: lastQueueRequests,
616
+ };
617
+ const queueAbort = new AbortController();
618
+ let queueTimer: NodeJS.Timeout | undefined;
619
+ let queueSnapshot: Promise<void> | null = null;
620
+ const stopQueueObservation = (): void => {
621
+ admissionWait = stepAdmissionWait(admissionWait, { kind: 'stop', at: performance.now() });
622
+ clearInterval(queueTimer);
623
+ queueAbort.abort();
624
+ };
567
625
  // Records the run's verdict when it has a task to record as. An `expiry` is set only when a bound ended the run.
568
- // An `operational` run failed on its CPU pool (a sub-pool that did not close): an ordinary failure, whatever bound also
569
- // fired, so it is never judged as a bound.
626
+ // Observation and ownership failures remain operational even after valid queue evidence; cancellation alone does not.
570
627
  const recordRun = async (
571
628
  exitCode: number,
572
629
  expiry: ExpiredBound | null,
@@ -579,10 +636,37 @@ export async function runBoundedExec(
579
636
  return;
580
637
  }
581
638
  const { task, directory: recordDirectory } = context.record;
582
- const report = await readJunitReport(recordDirectory);
583
- const verdict = operational
584
- ? judgeOperationalFailure(exitCode, report)
585
- : judgeRun({ exitCode, elapsedMs, expiry, report, noSpace, lockWaits });
639
+ const report = await readJunitReport(
640
+ recordDirectory,
641
+ requested.search(BUN_TEST) !== -1 && resultWriter !== null
642
+ ? (results) => {
643
+ const error = resultWriter?.reconcile(results) ?? null;
644
+ if (error !== null) {
645
+ resultFailure ??= error;
646
+ appendStderr(`bounded-exec: ${error}\n`);
647
+ }
648
+ }
649
+ : undefined,
650
+ ).finally(async () => {
651
+ const error = (await resultWriter?.close()) ?? null;
652
+ if (error !== null) {
653
+ resultFailure ??= error;
654
+ appendStderr(`bounded-exec: ${error}\n`);
655
+ }
656
+ });
657
+ const admissionQueue = queueWait();
658
+ const verdict =
659
+ operational || resultFailure !== null
660
+ ? judgeOperationalFailure(exitCode, report)
661
+ : judgeRun({
662
+ exitCode,
663
+ elapsedMs,
664
+ expiry,
665
+ report,
666
+ noSpace,
667
+ lockWaits,
668
+ admissionQueue,
669
+ });
586
670
  if (verdict.outcome === 'bound' && verdict.bound === 'test') {
587
671
  appendStderr(`Every failing test failed only on its per-test timeout: ${verdict.tests.join(', ')}\n`);
588
672
  }
@@ -616,10 +700,11 @@ export async function runBoundedExec(
616
700
  }
617
701
  pool = created.pool;
618
702
  }
703
+ const native = directory === null ? { command: sized.command, env: {} } : withTestResultReport(sized.command);
619
704
  const command =
620
705
  directory === null
621
- ? withProgressReport(sized.command)
622
- : await withJunitReport(withProgressReport(sized.command), directory);
706
+ ? withProgressReport(native.command)
707
+ : await withJunitReport(withProgressReport(native.command), directory);
623
708
  // A runner bounded-exec cannot see in the command (a script that spawns `bun test`) writes its JUnit report to this
624
709
  // path when it is set. The grant's sizing outranks the target's own environment: the runner may use what it was
625
710
  // granted, not what it was configured with. The sub-pool's environment comes last: a descendant reads only the pool
@@ -628,7 +713,9 @@ export async function runBoundedExec(
628
713
  ...(directory === null ? {} : { [JUNIT_REPORT_ENV]: join(directory, JUNIT_FILE) }),
629
714
  ...targetEnv(options.env, appendStderr),
630
715
  ...sized.env,
716
+ ...native.env,
631
717
  ...(pool?.environment ?? {}),
718
+ ...(leaf?.kind === 'cargo' ? { MBX_GRANT_FD: String(MBX_GRANT_FD) } : {}),
632
719
  };
633
720
  // A fixture rides as segment 0 of the lifecycle. Without the channel (Windows) it is plain shell text ahead
634
721
  // of the command: the progress bound runs during it and its own ceiling does not.
@@ -645,26 +732,37 @@ export async function runBoundedExec(
645
732
  // With a supervisor, `cowshed subtree-root` asks it for a handle on itself, writes the answer to the handle
646
733
  // descriptor, and execs `/bin/sh -c command` in the same process: the command's pid, group and life are unchanged.
647
734
  const unadmitted = hooked?.command ?? (fixture === null ? command : `${fixture.command} && ${command}`);
648
- const shellCommand = leaf === null ? unadmitted : withMbxLeaf(unadmitted, leaf);
735
+ const shellCommand = withMbxLeaf(unadmitted, leaf);
736
+ // Only leaves known to be scheduled can exclude zero-hold time. Others keep lifecycle hang protection.
737
+ const queued = mbxIdleGated(leaf, sized.command, { ...process.env, ...env });
738
+ // fd 4 is unused without a subtree admission, so the grant takes fd 5 either way.
739
+ const grantSlots: IOType[] = leaf !== null ? ['ignore', 'pipe'] : [];
740
+ const rootGrantSlot: IOType[] = leaf !== null ? ['pipe'] : [];
649
741
  const root = owner?.rootArgv(shellCommand);
650
742
  let child: ChildProcess;
743
+ const results = directory === null ? null : await createTestResultWriter(directory, { cwd, root: context.root });
744
+ if (results !== null && !results.ok) return refuse(`bounded-exec: ${results.error}\n`);
745
+ resultWriter = results?.ok ? results.writer : null;
651
746
  try {
652
747
  // fd 3 is the runner lifecycle's duplex socket when the command is hooked; the subtree admission takes fd 4, which
653
- // `cowshed subtree-root` closes before it execs, so fd 3 reaches the shell unchanged.
748
+ // cowshed closes fd 4 before exec; fd 5 carries mbx's admission/release stream, including Cargo descendants.
654
749
  child =
655
750
  root === undefined
656
751
  ? spawn(shellCommand, [], {
657
752
  cwd,
658
753
  env: mergeEnv(lease?.directory ?? ordinaryTemp, env),
659
754
  shell: true,
660
- stdio: hooked === null ? 'pipe' : ['pipe', 'pipe', 'pipe', 'pipe'],
755
+ stdio:
756
+ hooked === null && leaf === null
757
+ ? 'pipe'
758
+ : ['pipe', 'pipe', 'pipe', hooked === null ? 'ignore' : 'pipe', ...grantSlots],
661
759
  detached: process.platform !== 'win32',
662
760
  windowsHide: true,
663
761
  })
664
762
  : spawn(root.file, root.args, {
665
763
  cwd,
666
764
  env: mergeEnv(lease?.directory ?? ordinaryTemp, env),
667
- stdio: ['pipe', 'pipe', 'pipe', hooked === null ? 'ignore' : 'pipe', 'pipe'],
765
+ stdio: ['pipe', 'pipe', 'pipe', hooked === null ? 'ignore' : 'pipe', 'pipe', ...rootGrantSlot],
668
766
  detached: process.platform !== 'win32',
669
767
  windowsHide: true,
670
768
  });
@@ -688,7 +786,20 @@ export async function runBoundedExec(
688
786
  }
689
787
 
690
788
  // Progress is a finished test, not output: a runner's SLOW warnings and a hung test's heartbeat keep printing.
691
- const progress = createRunProgress(command);
789
+ const progress = createRunProgress(
790
+ command,
791
+ resultWriter === null
792
+ ? undefined
793
+ : (result) => {
794
+ const error = resultWriter?.append(result) ?? null;
795
+ if (error !== null && resultFailure === null) {
796
+ resultFailure = error;
797
+ lifecycleFailure ??= error;
798
+ appendStderr(`bounded-exec: ${error}\n`);
799
+ termination = terminate();
800
+ }
801
+ },
802
+ );
692
803
  // A write that found no space is said once, in the output, by whatever made it; the volume is read when it is said,
693
804
  // since the writer usually removes what it half-wrote before it exits and the volume no longer looks full by then.
694
805
  const noSpace = createNoSpaceWatch();
@@ -758,6 +869,41 @@ export async function runBoundedExec(
758
869
  : kind === 'fixture'
759
870
  ? { kind, limitMs: fixture?.timeoutMs ?? 0 }
760
871
  : { kind, limitMs: timeoutMs ?? 0 };
872
+ let checkingIdle = false;
873
+ const armBoundTimer = (): void => {
874
+ clearTimeout(boundTimer);
875
+ if (state.settled || state.expiry !== null) return;
876
+ const next = earliestBound(checkingIdle ? { ...bounds, idle: null } : bounds);
877
+ if (next !== null) {
878
+ boundTimer = setTimeout(
879
+ () => {
880
+ void confirmExpiry(next.kind, next.at);
881
+ },
882
+ Math.max(0, next.at - Date.now()),
883
+ );
884
+ }
885
+ };
886
+ const confirmExpiry = async (kind: BoundKind, at: number): Promise<void> => {
887
+ if (state.settled || state.expiry !== null) return;
888
+ const scheduled = earliestBound(checkingIdle ? { ...bounds, idle: null } : bounds);
889
+ if (scheduled?.kind !== kind || scheduled.at !== at) return;
890
+ if (kind === 'idle' && leaf !== null) {
891
+ if (checkingIdle) return;
892
+ checkingIdle = true;
893
+ armBoundTimer();
894
+ try {
895
+ // Drain an older poll, then take a fresh bounded snapshot before idle can end queued work. Total and fixture
896
+ // ceilings remain armed while this observation runs.
897
+ await refreshQueue(true);
898
+ } finally {
899
+ checkingIdle = false;
900
+ }
901
+ }
902
+ if (state.settled || state.expiry !== null) return;
903
+ const next = earliestBound(bounds);
904
+ if (next !== null && next.at <= Date.now()) expire(expiredAs(next.kind));
905
+ else armBoundTimer();
906
+ };
761
907
  const stepTo = (event: BoundEvent): void => {
762
908
  if (state.settled || state.expiry !== null) return;
763
909
  const idleBefore = bounds.idle;
@@ -775,9 +921,75 @@ export async function runBoundedExec(
775
921
  idleCapture = undefined;
776
922
  if (bounds.idle !== null) scheduleCapture(bounds.idle);
777
923
  }
778
- clearTimeout(boundTimer);
779
- const next = earliestBound(bounds);
780
- if (next !== null) boundTimer = setTimeout(() => expire(expiredAs(next.kind)), Math.max(0, next.at - Date.now()));
924
+ armBoundTimer();
925
+ };
926
+
927
+ let lastQueueDiagnostic = '';
928
+ let lastQueueDiagnosticAt: number | null = null;
929
+ let lastSnapshotDiagnostics = '';
930
+ const unconfirmAdmission = (detail: string): void => {
931
+ if (state.settled || state.expiry !== null || queueAbort.signal.aborted) return;
932
+ if (unconfirmedAdmission !== detail) {
933
+ tell(
934
+ `bounded-exec: mbx admission snapshot failed: ${detail}; admission is unconfirmed, so this wait earns no idle credit until a snapshot succeeds\n`,
935
+ );
936
+ }
937
+ unconfirmedAdmission = detail;
938
+ lastQueueRequests = [];
939
+ admissionWait = stepAdmissionWait(admissionWait, { kind: 'snapshot', at: performance.now(), waiting: false });
940
+ // The wait since the last confirmed snapshot is unconfirmed: the idle bound resumes from that snapshot.
941
+ stepTo({ kind: 'admission-queue', at: confirmedAdmissionAt ?? Date.now(), waiting: false, testing: isTesting() });
942
+ };
943
+ const refreshQueue = async (fresh: boolean): Promise<void> => {
944
+ if (fresh && queueSnapshot !== null) await queueSnapshot;
945
+ if (state.settled || state.expiry !== null || queueAbort.signal.aborted || child.pid === undefined) return;
946
+ const pid = child.pid;
947
+ queueSnapshot ??= (async () => {
948
+ const snapshot = await snapshotMbxQueue(
949
+ pid,
950
+ mergeEnv(lease?.directory ?? ordinaryTemp, env),
951
+ queueAbort.signal,
952
+ queueHost,
953
+ );
954
+ if (state.settled || state.expiry !== null || queueAbort.signal.aborted) return;
955
+ if (!snapshot.ok) {
956
+ unconfirmAdmission(snapshot.detail);
957
+ return;
958
+ }
959
+ const diagnostics = snapshot.diagnostics?.join('; ') ?? '';
960
+ if (diagnostics !== '' && diagnostics !== lastSnapshotDiagnostics) tell(`bounded-exec: ${diagnostics}\n`);
961
+ lastSnapshotDiagnostics = diagnostics;
962
+ const waiting = snapshot.requests.length > 0;
963
+ const observedAt = Date.now();
964
+ const observedWaitAt = performance.now();
965
+ admissionWait = stepAdmissionWait(admissionWait, { kind: 'snapshot', at: observedWaitAt, waiting });
966
+ unconfirmedAdmission = null;
967
+ confirmedAdmissionAt = observedAt;
968
+ if (waiting) {
969
+ lastQueueRequests = snapshot.requests;
970
+ const positions = JSON.stringify(snapshot.requests);
971
+ if (
972
+ positions !== lastQueueDiagnostic ||
973
+ lastQueueDiagnosticAt === null ||
974
+ Date.now() - lastQueueDiagnosticAt >= HANG_CAPTURE_MS
975
+ ) {
976
+ const waitedMs = admissionWaitMs(admissionWait, observedWaitAt);
977
+ tell(`cowshed: mbx ${describeAdmissionQueue({ waitedMs, requests: snapshot.requests })}\n`);
978
+ lastQueueDiagnostic = positions;
979
+ lastQueueDiagnosticAt = Date.now();
980
+ }
981
+ } else {
982
+ lastQueueDiagnostic = '';
983
+ }
984
+ stepTo({ kind: 'admission-queue', at: observedAt, waiting, testing: isTesting() });
985
+ })()
986
+ .catch((error: unknown) => {
987
+ unconfirmAdmission(error instanceof Error ? error.message : String(error));
988
+ })
989
+ .finally(() => {
990
+ queueSnapshot = null;
991
+ });
992
+ await queueSnapshot;
781
993
  };
782
994
 
783
995
  // Only the CHILD's own output is read for progress. The diagnostics below go
@@ -786,9 +998,15 @@ export async function runBoundedExec(
786
998
  const updateIdle = (finished: boolean, wasTesting: boolean): void => {
787
999
  stepTo({ kind: 'progress', at: Date.now(), testing: isTesting(), wasTesting, finished });
788
1000
  };
789
- const scanProgress = (stream: 'stdout' | 'stderr', text: string): void => {
1001
+ const scanProgress = (stream: 'stdout' | 'stderr', text: Buffer | string): void => {
790
1002
  if (!fixtureDone) return;
791
1003
  updateIdle(progress.scan(stream, text), isTesting());
1004
+ const error = progress.error();
1005
+ if (error !== null && lifecycleFailure === null) {
1006
+ lifecycleFailure = error;
1007
+ appendStderr(`bounded-exec: ${error}\n`);
1008
+ termination = terminate();
1009
+ }
792
1010
  };
793
1011
  if (lifecycle !== null) {
794
1012
  const channel = child.stdio[3];
@@ -826,14 +1044,14 @@ export async function runBoundedExec(
826
1044
  }
827
1045
  child.stdout?.on('data', (chunk: Buffer | string) => {
828
1046
  const text = chunk.toString();
829
- scanProgress('stdout', text);
1047
+ scanProgress('stdout', chunk);
830
1048
  const said = noSpace.scan('stdout', text);
831
1049
  if (said !== null) state.noSpace = sight(said);
832
1050
  appendStdout(text);
833
1051
  });
834
1052
  child.stderr?.on('data', (chunk: Buffer | string) => {
835
1053
  const text = chunk.toString();
836
- scanProgress('stderr', text);
1054
+ scanProgress('stderr', chunk);
837
1055
  const said = noSpace.scan('stderr', text);
838
1056
  if (said !== null) state.noSpace = sight(said);
839
1057
  appendStderr(text);
@@ -857,12 +1075,30 @@ export async function runBoundedExec(
857
1075
  const onProcessExit = (): void => {
858
1076
  void killChildTree();
859
1077
  };
1078
+ let signalCleanup: Promise<void> | null = null;
860
1079
  const onTerminationSignal = (signal: NodeJS.Signals): void => {
861
1080
  removeSignalHandlers();
862
1081
  cancelCapture();
863
- void killChildTree()
1082
+ clearTimeout(boundTimer);
1083
+ // Preserve explicit live membership before cancellation removes the queued process. A failed bounded observation
1084
+ // leaves the admission unconfirmed, never a manufactured queue or admission.
1085
+ signalCleanup = (leaf === null ? Promise.resolve() : refreshQueue(false))
1086
+ .finally(() => {
1087
+ stopQueueObservation();
1088
+ return killChildTree();
1089
+ })
864
1090
  .finally(releaseCpu)
865
1091
  .finally(() => releaseLease(true))
1092
+ .finally(async () =>
1093
+ recordRun(
1094
+ signalToExitCode(signal),
1095
+ state.expiry,
1096
+ Date.now() - startedAt,
1097
+ state.poolCloseError !== null || lifecycleFailure !== null || state.retireError !== null,
1098
+ state.noSpace === null ? null : await state.noSpace,
1099
+ state.lockWaits,
1100
+ ),
1101
+ )
866
1102
  .finally(() => process.kill(process.pid, signal));
867
1103
  };
868
1104
  const onSigint = (): void => onTerminationSignal('SIGINT');
@@ -888,10 +1124,12 @@ export async function runBoundedExec(
888
1124
  child.once('error', (error) => {
889
1125
  appendStderr(`${error.message}\n`);
890
1126
  state.settled = true;
1127
+ stopQueueObservation();
891
1128
  resolve({ code: 1, signal: null });
892
1129
  });
893
1130
  child.once(hooked === null ? 'exit' : 'close', (code: number | null, signal: NodeJS.Signals | null) => {
894
1131
  state.settled = true;
1132
+ stopQueueObservation();
895
1133
  resolve({ code, signal });
896
1134
  });
897
1135
  });
@@ -914,7 +1152,11 @@ export async function runBoundedExec(
914
1152
  bound.tests.length > 0
915
1153
  ? `still running: ${bound.tests.join(', ')}`
916
1154
  : 'the runner named no test still running'
917
- } (cwd=${cwd}): ${command}\n`
1155
+ } (cwd=${cwd}): ${command}\n${
1156
+ unconfirmedAdmission === null
1157
+ ? ''
1158
+ : `The last mbx admission snapshot failed (${unconfirmedAdmission}), so the wait it could not confirm earned no idle credit\n`
1159
+ }`
918
1160
  : bound.kind === 'fixture'
919
1161
  ? `\nFixture command timed out after ${elapsedMs}ms (fixtureTimeoutMs=${bound.limitMs}, cwd=${cwd}): ${fixture?.command ?? command}\n`
920
1162
  : `\nCommand timed out after ${elapsedMs}ms (timeoutMs=${bound.limitMs}, cwd=${cwd}): ${command}\n`,
@@ -1010,7 +1252,54 @@ export async function runBoundedExec(
1010
1252
 
1011
1253
  // The fixture's ceiling lives and dies with its segment: the segment's terminal clears it, so it can never
1012
1254
  // fire on a command the fixture already handed over to.
1013
- stepTo({ kind: 'spawn', at: Date.now(), testing: isTesting(), fixture: fixture !== null && lifecycle !== null });
1255
+ stepTo({
1256
+ kind: 'spawn',
1257
+ at: Date.now(),
1258
+ testing: isTesting(),
1259
+ fixture: fixture !== null && lifecycle !== null,
1260
+ queued,
1261
+ });
1262
+ // Read every byte after spawn accounting: a chunk can contain several grants and releases. EOF is neither.
1263
+ if (leaf !== null) {
1264
+ const channel = child.stdio.at(MBX_GRANT_FD);
1265
+ if (!(channel instanceof Readable)) throw new Error('an mbx leaf requires its grant descriptor');
1266
+ let said = false;
1267
+ const failGrant = (detail: string): void => {
1268
+ if (lifecycleFailure !== null || state.expiry !== null || state.settled) return;
1269
+ lifecycleFailure ??= `mbx grant descriptor: ${detail}`;
1270
+ appendStderr(`bounded-exec: ${lifecycleFailure}\n`);
1271
+ termination = terminate();
1272
+ };
1273
+ channel.on('data', (chunk: Buffer) => {
1274
+ for (const byte of chunk) {
1275
+ if (byte === 103) {
1276
+ if (!said) {
1277
+ mbxAdmitted = true;
1278
+ said = true;
1279
+ tell(
1280
+ leaf.kind === 'reserve'
1281
+ ? `cowshed: mbx reserve granted elapsed=${Date.now() - startedAt}ms asked=${leaf.demand.want} priority=${leaf.priority}\n`
1282
+ : `cowshed: mbx cargo granted elapsed=${Date.now() - startedAt}ms\n`,
1283
+ );
1284
+ }
1285
+ stepTo({ kind: 'granted', at: Date.now(), testing: isTesting(), fixture: !fixtureDone });
1286
+ } else if (byte === 114) {
1287
+ stepTo({ kind: 'released', at: Date.now() });
1288
+ } else {
1289
+ failGrant(`unexpected byte ${byte}`);
1290
+ return;
1291
+ }
1292
+ }
1293
+ });
1294
+ channel.on('error', (error) => failGrant(error.message));
1295
+ }
1296
+
1297
+ if (leaf !== null) {
1298
+ queueTimer = setInterval(() => {
1299
+ void refreshQueue(false);
1300
+ }, MBX_QUEUE_POLL_MS);
1301
+ void refreshQueue(false);
1302
+ }
1014
1303
 
1015
1304
  // A grant's connection is a ledger entry, not the command's owner: the lending parent's exit ends it, and what runs
1016
1305
  // under it runs on over budget until it finishes. Its close, while the command runs, is said and changes nothing
@@ -1029,6 +1318,12 @@ export async function runBoundedExec(
1029
1318
  }
1030
1319
 
1031
1320
  const { code: exitCode, signal: exitSignal } = await exitPromise;
1321
+ if (signalCleanup !== null) {
1322
+ await signalCleanup;
1323
+ return { success: false, terminalOutput: outputChunks.join('') };
1324
+ }
1325
+ stopQueueObservation();
1326
+ await queueSnapshot;
1032
1327
  await termination;
1033
1328
  clearTimeout(boundTimer);
1034
1329
  cancelCapture();
@@ -1069,7 +1364,7 @@ export async function runBoundedExec(
1069
1364
  const operationalFailure = state.poolCloseError !== null || lifecycleFailure !== null || state.retireError !== null;
1070
1365
  await recordRun(code, state.expiry, Date.now() - startedAt, operationalFailure, seen, state.lockWaits);
1071
1366
  return {
1072
- success: !operationalFailure && state.expiry === null && code === 0,
1367
+ success: !operationalFailure && resultFailure === null && state.expiry === null && code === 0,
1073
1368
  terminalOutput: outputChunks.join(''),
1074
1369
  };
1075
1370
  }
@@ -383,11 +383,12 @@ export function stopProbe(file: string, pid: number): boolean {
383
383
  }
384
384
 
385
385
  /** A probe owns its group and output pipes; its deadline never waits for an unresponsive child to reap. */
386
- function runProbe(
386
+ export function runProbe(
387
387
  file: string,
388
388
  args: readonly string[],
389
389
  timeoutMs: number,
390
390
  signal: AbortSignal,
391
+ environment?: NodeJS.ProcessEnv,
391
392
  ): Promise<HangProbeResult> {
392
393
  // Keep the package's es2022-compatible event-completion convention.
393
394
  return new Promise((resolve) => {
@@ -395,7 +396,7 @@ function runProbe(
395
396
  resolve({ status: 'timed-out', detail: 'capture deadline already reached', stdout: '', stderr: '' });
396
397
  return;
397
398
  }
398
- const child = spawn(file, [...args], { detached: true, stdio: ['ignore', 'pipe', 'pipe'] });
399
+ const child = spawn(file, [...args], { detached: true, stdio: ['ignore', 'pipe', 'pipe'], env: environment });
399
400
  let stdout = '';
400
401
  let stderr = '';
401
402
  let bytes = 0;