@smoothbricks/nx-plugin 0.6.2 → 0.6.3-0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. package/README.md +281 -73
  2. package/dist/.build-inputs.json +20 -14
  3. package/dist/bin/smoo-nx-publish.d.ts +3 -0
  4. package/dist/bin/smoo-nx-publish.d.ts.map +1 -0
  5. package/dist/bin/smoo-nx-publish.js +3 -0
  6. package/dist/cargo-retention-hook.d.ts +6 -2
  7. package/dist/cargo-retention-hook.d.ts.map +1 -1
  8. package/dist/cargo-retention-hook.js +13 -7
  9. package/dist/eslint-file-set.d.ts +4 -1
  10. package/dist/eslint-file-set.d.ts.map +1 -1
  11. package/dist/eslint-sources.d.ts +25 -0
  12. package/dist/eslint-sources.d.ts.map +1 -0
  13. package/dist/eslint-sources.js +99 -0
  14. package/dist/executors/all-verdicts/executor.d.ts +2 -1
  15. package/dist/executors/all-verdicts/executor.d.ts.map +1 -1
  16. package/dist/executors/all-verdicts/executor.js +48 -2
  17. package/dist/executors/all-verdicts/schema.d.ts +13 -0
  18. package/dist/executors/all-verdicts/schema.d.ts.map +1 -1
  19. package/dist/executors/all-verdicts/schema.js +7 -0
  20. package/dist/executors/bounded-exec/bounds.d.ts +57 -8
  21. package/dist/executors/bounded-exec/bounds.d.ts.map +1 -1
  22. package/dist/executors/bounded-exec/bounds.js +100 -11
  23. package/dist/executors/bounded-exec/executor.d.ts +2 -1
  24. package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
  25. package/dist/executors/bounded-exec/executor.js +270 -27
  26. package/dist/executors/bounded-exec/hang-diagnostics.d.ts +2 -0
  27. package/dist/executors/bounded-exec/hang-diagnostics.d.ts.map +1 -1
  28. package/dist/executors/bounded-exec/hang-diagnostics.js +2 -2
  29. package/dist/executors/bounded-exec/mbx-pool.d.ts +20 -3
  30. package/dist/executors/bounded-exec/mbx-pool.d.ts.map +1 -1
  31. package/dist/executors/bounded-exec/mbx-pool.js +84 -4
  32. package/dist/executors/bounded-exec/mbx-queue.d.ts +30 -0
  33. package/dist/executors/bounded-exec/mbx-queue.d.ts.map +1 -0
  34. package/dist/executors/bounded-exec/mbx-queue.js +229 -0
  35. package/dist/executors/bounded-exec/progress.d.ts +9 -2
  36. package/dist/executors/bounded-exec/progress.d.ts.map +1 -1
  37. package/dist/executors/bounded-exec/progress.js +148 -22
  38. package/dist/executors/bounded-exec/test-result-writer.d.ts +30 -0
  39. package/dist/executors/bounded-exec/test-result-writer.d.ts.map +1 -0
  40. package/dist/executors/bounded-exec/test-result-writer.js +126 -0
  41. package/dist/executors/bounded-exec/test-results.d.ts +33 -0
  42. package/dist/executors/bounded-exec/test-results.d.ts.map +1 -0
  43. package/dist/executors/bounded-exec/test-results.js +33 -0
  44. package/dist/executors/bounded-exec/verdict.d.ts +20 -2
  45. package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
  46. package/dist/executors/bounded-exec/verdict.js +102 -4
  47. package/dist/index.d.ts +3 -2
  48. package/dist/index.d.ts.map +1 -1
  49. package/dist/index.js +67 -65
  50. package/dist/nx-publish.d.ts +24 -0
  51. package/dist/nx-publish.d.ts.map +1 -0
  52. package/dist/nx-publish.js +150 -0
  53. package/dist/runtime-input-policy.d.ts +14 -0
  54. package/dist/runtime-input-policy.d.ts.map +1 -1
  55. package/dist/runtime-input-policy.js +1 -1
  56. package/dist/tool-bins.d.ts +6 -8
  57. package/dist/tool-bins.d.ts.map +1 -1
  58. package/dist/tool-bins.js +7 -5
  59. package/dist/workspace-index.d.ts +7 -0
  60. package/dist/workspace-index.d.ts.map +1 -1
  61. package/dist/workspace-index.js +11 -0
  62. package/executors.json +1 -1
  63. package/managed/raw/patches/ttsc@0.34.1.patch +75 -0
  64. package/managed/raw/tooling/devenv +1 -2
  65. package/managed/raw/tooling/direnv/developer-links.ts +3 -0
  66. package/managed/raw/tooling/direnv/devenv.smoo.nix +34 -15
  67. package/managed/raw/tooling/direnv/github-actions-bootstrap.sh +20 -10
  68. package/managed/raw/tooling/direnv/setup-environment.ts +35 -20
  69. package/managed/raw/tooling/direnv/ttsc-env.sh +8 -6
  70. package/managed/templates/github/actions/cache-node-modules/action.yml +1 -1
  71. package/managed/templates/github/actions/cache-ttsc-plugins/action.yml +12 -11
  72. package/package.json +223 -7
  73. package/runtime/nx-publish/cache_adapter.js +45 -0
  74. package/runtime/nx-publish/nx_publish.js +60 -0
  75. package/runtime-input.sh +12 -3
  76. package/src/bin/smoo-nx-publish.ts +4 -0
  77. package/src/cargo-retention-hook.ts +16 -10
  78. package/src/eslint-file-set.ts +4 -1
  79. package/src/eslint-sources.ts +134 -0
  80. package/src/executors/all-verdicts/executor.ts +58 -3
  81. package/src/executors/all-verdicts/schema.json +4 -0
  82. package/src/executors/all-verdicts/schema.ts +14 -0
  83. package/src/executors/bounded-exec/bounds.ts +132 -15
  84. package/src/executors/bounded-exec/executor.ts +303 -27
  85. package/src/executors/bounded-exec/hang-diagnostics.ts +3 -2
  86. package/src/executors/bounded-exec/mbx-pool.ts +82 -5
  87. package/src/executors/bounded-exec/mbx-queue.ts +170 -0
  88. package/src/executors/bounded-exec/progress.ts +168 -15
  89. package/src/executors/bounded-exec/test-result-writer.ts +152 -0
  90. package/src/executors/bounded-exec/test-results.ts +39 -0
  91. package/src/executors/bounded-exec/verdict.ts +108 -5
  92. package/src/index.ts +74 -70
  93. package/src/nx-publish.ts +157 -0
  94. package/src/runtime-input-policy.ts +1 -1
  95. package/src/tool-bins.ts +13 -9
  96. package/src/workspace-index.ts +12 -0
  97. package/test-timings.json +8 -3
@@ -1,4 +1,4 @@
1
- import { type ChildProcess, spawn } from 'node:child_process';
1
+ import { type ChildProcess, type IOType, spawn } from 'node:child_process';
2
2
  import { accessSync, constants } from 'node:fs';
3
3
  import { mkdtemp, rm } from 'node:fs/promises';
4
4
  import { availableParallelism, userInfo } from 'node:os';
@@ -13,7 +13,17 @@ import {
13
13
  processGroupHasMembers,
14
14
  signalProcessGroup,
15
15
  } from '../../process-group.js';
16
- import { type BoundEvent, type BoundKind, type Bounds, earliestBound, stepBounds, UNARMED } from './bounds.js';
16
+ import {
17
+ admissionWaitMs,
18
+ type BoundEvent,
19
+ type BoundKind,
20
+ type Bounds,
21
+ earliestBound,
22
+ NO_ADMISSION_WAIT,
23
+ stepAdmissionWait,
24
+ stepBounds,
25
+ UNARMED,
26
+ } from './bounds.js';
17
27
  import {
18
28
  CPU_POOL_ENV,
19
29
  type CpuPool,
@@ -28,7 +38,8 @@ import type { HangDiagnostics } from './hang-diagnostics.js';
28
38
  import { captureHangDiagnostics, HANG_CAPTURE_MS } from './hang-diagnostics.js';
29
39
  import type { CpuWaitRow } from './lease-wait.js';
30
40
  import type { LockWait } from './lock-wait.js';
31
- import { mbxLeaf, mbxPool, withMbxLeaf } from './mbx-pool.js';
41
+ import { MBX_GRANT_FD, mbxIdleGated, mbxLeaf, mbxPool, withMbxLeaf } from './mbx-pool.js';
42
+ import { MBX_QUEUE_POLL_MS, type MbxQueuedRequest, snapshotMbxQueue } from './mbx-queue.js';
32
43
  import { createNoSpaceWatch } from './no-space.js';
33
44
  import {
34
45
  COWSHED_EXECUTABLE_ENV,
@@ -39,7 +50,7 @@ import {
39
50
  type SubtreeAdmission,
40
51
  type SubtreeOwner,
41
52
  } from './owned-subtree.js';
42
- import { createRunProgress, withProgressReport } from './progress.js';
53
+ import { createRunProgress, withProgressReport, withTestResultReport } from './progress.js';
43
54
  import {
44
55
  createHostCommands,
45
56
  describeCoverageGap,
@@ -56,10 +67,14 @@ import { createRunnerLifecycle, withFixtureLifecycle, withRunnerLifecycle } from
56
67
  import { CPU_TOKENS_ENV, runnerDemand, sizeRunner } from './runner-sizing.js';
57
68
  import { checkRuntimePin, describeRuntimePinRefusal } from './runtime-pin.js';
58
69
  import type { BoundedExecOptions } from './schema.js';
70
+ import { createTestResultWriter, type TestResultWriter } from './test-result-writer.js';
59
71
  import {
72
+ type AdmissionQueueWait,
60
73
  type BoundedExecRecord,
61
74
  type BoundedExecTask,
75
+ BUN_TEST,
62
76
  clearTaskDirectory,
77
+ describeAdmissionQueue,
63
78
  describeNoSpace,
64
79
  type ExpiredBound,
65
80
  JUNIT_FILE,
@@ -291,6 +306,7 @@ export async function runBoundedExec(
291
306
  inherited: InheritedCpuPool | null = null,
292
307
  captureDiagnostics: typeof captureHangDiagnostics = captureHangDiagnostics,
293
308
  owner: SubtreeOwner | null = null,
309
+ queueHost?: Parameters<typeof snapshotMbxQueue>[3],
294
310
  ): Promise<BoundedExecResult> {
295
311
  const cwd = resolveCwd(options.cwd, context.root);
296
312
  const directory = context.record?.directory ?? null;
@@ -307,6 +323,8 @@ export async function runBoundedExec(
307
323
  ? { command: options.fixtureCommand, timeoutMs: options.fixtureTimeoutMs }
308
324
  : null;
309
325
  const outputChunks: string[] = [];
326
+ let resultWriter: TestResultWriter | null = null;
327
+ let resultFailure: string | null = null;
310
328
  const state: RunState = {
311
329
  settled: false,
312
330
  expiry: null,
@@ -564,9 +582,27 @@ export async function runBoundedExec(
564
582
  })();
565
583
  return cpuReleased;
566
584
  };
585
+ let mbxAdmitted = false;
586
+ let admissionWait = NO_ADMISSION_WAIT;
587
+ let lastQueueRequests: readonly MbxQueuedRequest[] = [];
588
+ let queueFailure: string | null = null;
589
+ const queueWait = (): AdmissionQueueWait | null =>
590
+ mbxAdmitted || lastQueueRequests.length === 0
591
+ ? null
592
+ : {
593
+ waitedMs: admissionWaitMs(admissionWait, performance.now()),
594
+ requests: lastQueueRequests,
595
+ };
596
+ const queueAbort = new AbortController();
597
+ let queueTimer: NodeJS.Timeout | undefined;
598
+ let queueSnapshot: Promise<void> | null = null;
599
+ const stopQueueObservation = (): void => {
600
+ admissionWait = stepAdmissionWait(admissionWait, { kind: 'stop', at: performance.now() });
601
+ clearInterval(queueTimer);
602
+ queueAbort.abort();
603
+ };
567
604
  // Records the run's verdict when it has a task to record as. An `expiry` is set only when a bound ended the run.
568
- // An `operational` run failed on its CPU pool (a sub-pool that did not close): an ordinary failure, whatever bound also
569
- // fired, so it is never judged as a bound.
605
+ // Observation and ownership failures remain operational even after valid queue evidence; cancellation alone does not.
570
606
  const recordRun = async (
571
607
  exitCode: number,
572
608
  expiry: ExpiredBound | null,
@@ -579,10 +615,37 @@ export async function runBoundedExec(
579
615
  return;
580
616
  }
581
617
  const { task, directory: recordDirectory } = context.record;
582
- const report = await readJunitReport(recordDirectory);
583
- const verdict = operational
584
- ? judgeOperationalFailure(exitCode, report)
585
- : judgeRun({ exitCode, elapsedMs, expiry, report, noSpace, lockWaits });
618
+ const report = await readJunitReport(
619
+ recordDirectory,
620
+ requested.search(BUN_TEST) !== -1 && resultWriter !== null
621
+ ? (results) => {
622
+ const error = resultWriter?.reconcile(results) ?? null;
623
+ if (error !== null) {
624
+ resultFailure ??= error;
625
+ appendStderr(`bounded-exec: ${error}\n`);
626
+ }
627
+ }
628
+ : undefined,
629
+ ).finally(async () => {
630
+ const error = (await resultWriter?.close()) ?? null;
631
+ if (error !== null) {
632
+ resultFailure ??= error;
633
+ appendStderr(`bounded-exec: ${error}\n`);
634
+ }
635
+ });
636
+ const admissionQueue = queueWait();
637
+ const verdict =
638
+ operational || resultFailure !== null
639
+ ? judgeOperationalFailure(exitCode, report)
640
+ : judgeRun({
641
+ exitCode,
642
+ elapsedMs,
643
+ expiry,
644
+ report,
645
+ noSpace,
646
+ lockWaits,
647
+ admissionQueue,
648
+ });
586
649
  if (verdict.outcome === 'bound' && verdict.bound === 'test') {
587
650
  appendStderr(`Every failing test failed only on its per-test timeout: ${verdict.tests.join(', ')}\n`);
588
651
  }
@@ -616,10 +679,11 @@ export async function runBoundedExec(
616
679
  }
617
680
  pool = created.pool;
618
681
  }
682
+ const native = directory === null ? { command: sized.command, env: {} } : withTestResultReport(sized.command);
619
683
  const command =
620
684
  directory === null
621
- ? withProgressReport(sized.command)
622
- : await withJunitReport(withProgressReport(sized.command), directory);
685
+ ? withProgressReport(native.command)
686
+ : await withJunitReport(withProgressReport(native.command), directory);
623
687
  // A runner bounded-exec cannot see in the command (a script that spawns `bun test`) writes its JUnit report to this
624
688
  // path when it is set. The grant's sizing outranks the target's own environment: the runner may use what it was
625
689
  // granted, not what it was configured with. The sub-pool's environment comes last: a descendant reads only the pool
@@ -628,7 +692,9 @@ export async function runBoundedExec(
628
692
  ...(directory === null ? {} : { [JUNIT_REPORT_ENV]: join(directory, JUNIT_FILE) }),
629
693
  ...targetEnv(options.env, appendStderr),
630
694
  ...sized.env,
695
+ ...native.env,
631
696
  ...(pool?.environment ?? {}),
697
+ ...(leaf?.kind === 'cargo' ? { MBX_GRANT_FD: String(MBX_GRANT_FD) } : {}),
632
698
  };
633
699
  // A fixture rides as segment 0 of the lifecycle. Without the channel (Windows) it is plain shell text ahead
634
700
  // of the command: the progress bound runs during it and its own ceiling does not.
@@ -646,25 +712,36 @@ export async function runBoundedExec(
646
712
  // descriptor, and execs `/bin/sh -c command` in the same process: the command's pid, group and life are unchanged.
647
713
  const unadmitted = hooked?.command ?? (fixture === null ? command : `${fixture.command} && ${command}`);
648
714
  const shellCommand = leaf === null ? unadmitted : withMbxLeaf(unadmitted, leaf);
715
+ // Only leaves known to be scheduled can exclude zero-hold time. Others keep lifecycle hang protection.
716
+ const queued = mbxIdleGated(leaf, sized.command, { ...process.env, ...env });
717
+ // fd 4 is unused without a subtree admission, so the grant takes fd 5 either way.
718
+ const grantSlots: IOType[] = leaf !== null ? ['ignore', 'pipe'] : [];
719
+ const rootGrantSlot: IOType[] = leaf !== null ? ['pipe'] : [];
649
720
  const root = owner?.rootArgv(shellCommand);
650
721
  let child: ChildProcess;
722
+ const results = directory === null ? null : await createTestResultWriter(directory, { cwd, root: context.root });
723
+ if (results !== null && !results.ok) return refuse(`bounded-exec: ${results.error}\n`);
724
+ resultWriter = results?.ok ? results.writer : null;
651
725
  try {
652
726
  // fd 3 is the runner lifecycle's duplex socket when the command is hooked; the subtree admission takes fd 4, which
653
- // `cowshed subtree-root` closes before it execs, so fd 3 reaches the shell unchanged.
727
+ // cowshed closes fd 4 before exec; fd 5 carries mbx's admission/release stream, including Cargo descendants.
654
728
  child =
655
729
  root === undefined
656
730
  ? spawn(shellCommand, [], {
657
731
  cwd,
658
732
  env: mergeEnv(lease?.directory ?? ordinaryTemp, env),
659
733
  shell: true,
660
- stdio: hooked === null ? 'pipe' : ['pipe', 'pipe', 'pipe', 'pipe'],
734
+ stdio:
735
+ hooked === null && leaf === null
736
+ ? 'pipe'
737
+ : ['pipe', 'pipe', 'pipe', hooked === null ? 'ignore' : 'pipe', ...grantSlots],
661
738
  detached: process.platform !== 'win32',
662
739
  windowsHide: true,
663
740
  })
664
741
  : spawn(root.file, root.args, {
665
742
  cwd,
666
743
  env: mergeEnv(lease?.directory ?? ordinaryTemp, env),
667
- stdio: ['pipe', 'pipe', 'pipe', hooked === null ? 'ignore' : 'pipe', 'pipe'],
744
+ stdio: ['pipe', 'pipe', 'pipe', hooked === null ? 'ignore' : 'pipe', 'pipe', ...rootGrantSlot],
668
745
  detached: process.platform !== 'win32',
669
746
  windowsHide: true,
670
747
  });
@@ -688,7 +765,20 @@ export async function runBoundedExec(
688
765
  }
689
766
 
690
767
  // Progress is a finished test, not output: a runner's SLOW warnings and a hung test's heartbeat keep printing.
691
- const progress = createRunProgress(command);
768
+ const progress = createRunProgress(
769
+ command,
770
+ resultWriter === null
771
+ ? undefined
772
+ : (result) => {
773
+ const error = resultWriter?.append(result) ?? null;
774
+ if (error !== null && resultFailure === null) {
775
+ resultFailure = error;
776
+ lifecycleFailure ??= error;
777
+ appendStderr(`bounded-exec: ${error}\n`);
778
+ termination = terminate();
779
+ }
780
+ },
781
+ );
692
782
  // A write that found no space is said once, in the output, by whatever made it; the volume is read when it is said,
693
783
  // since the writer usually removes what it half-wrote before it exits and the volume no longer looks full by then.
694
784
  const noSpace = createNoSpaceWatch();
@@ -758,8 +848,43 @@ export async function runBoundedExec(
758
848
  : kind === 'fixture'
759
849
  ? { kind, limitMs: fixture?.timeoutMs ?? 0 }
760
850
  : { kind, limitMs: timeoutMs ?? 0 };
851
+ let checkingIdle = false;
852
+ const armBoundTimer = (): void => {
853
+ clearTimeout(boundTimer);
854
+ if (state.settled || state.expiry !== null || queueFailure !== null) return;
855
+ const next = earliestBound(checkingIdle ? { ...bounds, idle: null } : bounds);
856
+ if (next !== null) {
857
+ boundTimer = setTimeout(
858
+ () => {
859
+ void confirmExpiry(next.kind, next.at);
860
+ },
861
+ Math.max(0, next.at - Date.now()),
862
+ );
863
+ }
864
+ };
865
+ const confirmExpiry = async (kind: BoundKind, at: number): Promise<void> => {
866
+ if (state.settled || state.expiry !== null || queueFailure !== null) return;
867
+ const scheduled = earliestBound(checkingIdle ? { ...bounds, idle: null } : bounds);
868
+ if (scheduled?.kind !== kind || scheduled.at !== at) return;
869
+ if (kind === 'idle' && leaf !== null) {
870
+ if (checkingIdle) return;
871
+ checkingIdle = true;
872
+ armBoundTimer();
873
+ try {
874
+ // Drain an older poll, then take a fresh bounded snapshot before idle can end queued work. Total and fixture
875
+ // ceilings remain armed while this observation runs.
876
+ await refreshQueue(true);
877
+ } finally {
878
+ checkingIdle = false;
879
+ }
880
+ }
881
+ if (state.settled || state.expiry !== null || queueFailure !== null) return;
882
+ const next = earliestBound(bounds);
883
+ if (next !== null && next.at <= Date.now()) expire(expiredAs(next.kind));
884
+ else armBoundTimer();
885
+ };
761
886
  const stepTo = (event: BoundEvent): void => {
762
- if (state.settled || state.expiry !== null) return;
887
+ if (state.settled || state.expiry !== null || queueFailure !== null) return;
763
888
  const idleBefore = bounds.idle;
764
889
  bounds = stepBounds(
765
890
  bounds,
@@ -775,9 +900,71 @@ export async function runBoundedExec(
775
900
  idleCapture = undefined;
776
901
  if (bounds.idle !== null) scheduleCapture(bounds.idle);
777
902
  }
903
+ armBoundTimer();
904
+ };
905
+
906
+ let lastQueueDiagnostic = '';
907
+ let lastQueueDiagnosticAt: number | null = null;
908
+ let lastSnapshotDiagnostics = '';
909
+ const failQueueObservation = (detail: string): void => {
910
+ if (state.settled || state.expiry !== null || queueAbort.signal.aborted || queueFailure !== null) return;
911
+ queueFailure = detail;
912
+ tell(
913
+ `bounded-exec: mbx admission snapshot failed: ${detail}; queue/admission state unconfirmed, not an idle wedge\n`,
914
+ );
778
915
  clearTimeout(boundTimer);
779
- const next = earliestBound(bounds);
780
- if (next !== null) boundTimer = setTimeout(() => expire(expiredAs(next.kind)), Math.max(0, next.at - Date.now()));
916
+ cancelCapture();
917
+ stopQueueObservation();
918
+ termination = terminate();
919
+ };
920
+ const refreshQueue = async (fresh: boolean): Promise<void> => {
921
+ if (fresh && queueSnapshot !== null) await queueSnapshot;
922
+ if (state.settled || state.expiry !== null || queueAbort.signal.aborted || child.pid === undefined) return;
923
+ const pid = child.pid;
924
+ queueSnapshot ??= (async () => {
925
+ const snapshot = await snapshotMbxQueue(
926
+ pid,
927
+ mergeEnv(lease?.directory ?? ordinaryTemp, env),
928
+ queueAbort.signal,
929
+ queueHost,
930
+ );
931
+ if (state.settled || state.expiry !== null || queueAbort.signal.aborted) return;
932
+ if (!snapshot.ok) {
933
+ failQueueObservation(snapshot.detail);
934
+ return;
935
+ }
936
+ const diagnostics = snapshot.diagnostics?.join('; ') ?? '';
937
+ if (diagnostics !== '' && diagnostics !== lastSnapshotDiagnostics) tell(`bounded-exec: ${diagnostics}\n`);
938
+ lastSnapshotDiagnostics = diagnostics;
939
+ const waiting = snapshot.requests.length > 0;
940
+ const observedAt = Date.now();
941
+ const observedWaitAt = performance.now();
942
+ admissionWait = stepAdmissionWait(admissionWait, { kind: 'snapshot', at: observedWaitAt, waiting });
943
+ if (waiting) {
944
+ lastQueueRequests = snapshot.requests;
945
+ const positions = JSON.stringify(snapshot.requests);
946
+ if (
947
+ positions !== lastQueueDiagnostic ||
948
+ lastQueueDiagnosticAt === null ||
949
+ Date.now() - lastQueueDiagnosticAt >= HANG_CAPTURE_MS
950
+ ) {
951
+ const waitedMs = admissionWaitMs(admissionWait, observedWaitAt);
952
+ tell(`cowshed: mbx ${describeAdmissionQueue({ waitedMs, requests: snapshot.requests })}\n`);
953
+ lastQueueDiagnostic = positions;
954
+ lastQueueDiagnosticAt = Date.now();
955
+ }
956
+ } else {
957
+ lastQueueDiagnostic = '';
958
+ }
959
+ stepTo({ kind: 'admission-queue', at: observedAt, waiting, testing: isTesting() });
960
+ })()
961
+ .catch((error: unknown) => {
962
+ failQueueObservation(error instanceof Error ? error.message : String(error));
963
+ })
964
+ .finally(() => {
965
+ queueSnapshot = null;
966
+ });
967
+ await queueSnapshot;
781
968
  };
782
969
 
783
970
  // Only the CHILD's own output is read for progress. The diagnostics below go
@@ -786,9 +973,15 @@ export async function runBoundedExec(
786
973
  const updateIdle = (finished: boolean, wasTesting: boolean): void => {
787
974
  stepTo({ kind: 'progress', at: Date.now(), testing: isTesting(), wasTesting, finished });
788
975
  };
789
- const scanProgress = (stream: 'stdout' | 'stderr', text: string): void => {
976
+ const scanProgress = (stream: 'stdout' | 'stderr', text: Buffer | string): void => {
790
977
  if (!fixtureDone) return;
791
978
  updateIdle(progress.scan(stream, text), isTesting());
979
+ const error = progress.error();
980
+ if (error !== null && lifecycleFailure === null) {
981
+ lifecycleFailure = error;
982
+ appendStderr(`bounded-exec: ${error}\n`);
983
+ termination = terminate();
984
+ }
792
985
  };
793
986
  if (lifecycle !== null) {
794
987
  const channel = child.stdio[3];
@@ -826,14 +1019,14 @@ export async function runBoundedExec(
826
1019
  }
827
1020
  child.stdout?.on('data', (chunk: Buffer | string) => {
828
1021
  const text = chunk.toString();
829
- scanProgress('stdout', text);
1022
+ scanProgress('stdout', chunk);
830
1023
  const said = noSpace.scan('stdout', text);
831
1024
  if (said !== null) state.noSpace = sight(said);
832
1025
  appendStdout(text);
833
1026
  });
834
1027
  child.stderr?.on('data', (chunk: Buffer | string) => {
835
1028
  const text = chunk.toString();
836
- scanProgress('stderr', text);
1029
+ scanProgress('stderr', chunk);
837
1030
  const said = noSpace.scan('stderr', text);
838
1031
  if (said !== null) state.noSpace = sight(said);
839
1032
  appendStderr(text);
@@ -857,12 +1050,33 @@ export async function runBoundedExec(
857
1050
  const onProcessExit = (): void => {
858
1051
  void killChildTree();
859
1052
  };
1053
+ let signalCleanup: Promise<void> | null = null;
860
1054
  const onTerminationSignal = (signal: NodeJS.Signals): void => {
861
1055
  removeSignalHandlers();
862
1056
  cancelCapture();
863
- void killChildTree()
1057
+ clearTimeout(boundTimer);
1058
+ // Preserve explicit live membership before cancellation removes the queued process. A failed bounded observation
1059
+ // reports an operational failure, never a manufactured queue or admission.
1060
+ signalCleanup = (leaf === null ? Promise.resolve() : refreshQueue(false))
1061
+ .finally(() => {
1062
+ stopQueueObservation();
1063
+ return killChildTree();
1064
+ })
864
1065
  .finally(releaseCpu)
865
1066
  .finally(() => releaseLease(true))
1067
+ .finally(async () =>
1068
+ recordRun(
1069
+ signalToExitCode(signal),
1070
+ state.expiry,
1071
+ Date.now() - startedAt,
1072
+ state.poolCloseError !== null ||
1073
+ lifecycleFailure !== null ||
1074
+ state.retireError !== null ||
1075
+ queueFailure !== null,
1076
+ state.noSpace === null ? null : await state.noSpace,
1077
+ state.lockWaits,
1078
+ ),
1079
+ )
866
1080
  .finally(() => process.kill(process.pid, signal));
867
1081
  };
868
1082
  const onSigint = (): void => onTerminationSignal('SIGINT');
@@ -888,10 +1102,12 @@ export async function runBoundedExec(
888
1102
  child.once('error', (error) => {
889
1103
  appendStderr(`${error.message}\n`);
890
1104
  state.settled = true;
1105
+ stopQueueObservation();
891
1106
  resolve({ code: 1, signal: null });
892
1107
  });
893
1108
  child.once(hooked === null ? 'exit' : 'close', (code: number | null, signal: NodeJS.Signals | null) => {
894
1109
  state.settled = true;
1110
+ stopQueueObservation();
895
1111
  resolve({ code, signal });
896
1112
  });
897
1113
  });
@@ -1010,7 +1226,54 @@ export async function runBoundedExec(
1010
1226
 
1011
1227
  // The fixture's ceiling lives and dies with its segment: the segment's terminal clears it, so it can never
1012
1228
  // fire on a command the fixture already handed over to.
1013
- stepTo({ kind: 'spawn', at: Date.now(), testing: isTesting(), fixture: fixture !== null && lifecycle !== null });
1229
+ stepTo({
1230
+ kind: 'spawn',
1231
+ at: Date.now(),
1232
+ testing: isTesting(),
1233
+ fixture: fixture !== null && lifecycle !== null,
1234
+ queued,
1235
+ });
1236
+ // Read every byte after spawn accounting: a chunk can contain several grants and releases. EOF is neither.
1237
+ if (leaf !== null) {
1238
+ const channel = child.stdio.at(MBX_GRANT_FD);
1239
+ if (!(channel instanceof Readable)) throw new Error('an mbx leaf requires its grant descriptor');
1240
+ let said = false;
1241
+ const failGrant = (detail: string): void => {
1242
+ if (lifecycleFailure !== null || state.expiry !== null || state.settled) return;
1243
+ lifecycleFailure ??= `mbx grant descriptor: ${detail}`;
1244
+ appendStderr(`bounded-exec: ${lifecycleFailure}\n`);
1245
+ termination = terminate();
1246
+ };
1247
+ channel.on('data', (chunk: Buffer) => {
1248
+ for (const byte of chunk) {
1249
+ if (byte === 103) {
1250
+ if (!said) {
1251
+ mbxAdmitted = true;
1252
+ said = true;
1253
+ tell(
1254
+ leaf.kind === 'reserve'
1255
+ ? `cowshed: mbx reserve granted elapsed=${Date.now() - startedAt}ms cpus=${leaf.cpus} priority=${leaf.priority}\n`
1256
+ : `cowshed: mbx cargo granted elapsed=${Date.now() - startedAt}ms\n`,
1257
+ );
1258
+ }
1259
+ stepTo({ kind: 'granted', at: Date.now(), testing: isTesting(), fixture: !fixtureDone });
1260
+ } else if (byte === 114) {
1261
+ stepTo({ kind: 'released', at: Date.now() });
1262
+ } else {
1263
+ failGrant(`unexpected byte ${byte}`);
1264
+ return;
1265
+ }
1266
+ }
1267
+ });
1268
+ channel.on('error', (error) => failGrant(error.message));
1269
+ }
1270
+
1271
+ if (leaf !== null) {
1272
+ queueTimer = setInterval(() => {
1273
+ void refreshQueue(false);
1274
+ }, MBX_QUEUE_POLL_MS);
1275
+ void refreshQueue(false);
1276
+ }
1014
1277
 
1015
1278
  // A grant's connection is a ledger entry, not the command's owner: the lending parent's exit ends it, and what runs
1016
1279
  // under it runs on over budget until it finishes. Its close, while the command runs, is said and changes nothing
@@ -1029,6 +1292,12 @@ export async function runBoundedExec(
1029
1292
  }
1030
1293
 
1031
1294
  const { code: exitCode, signal: exitSignal } = await exitPromise;
1295
+ if (signalCleanup !== null) {
1296
+ await signalCleanup;
1297
+ return { success: false, terminalOutput: outputChunks.join('') };
1298
+ }
1299
+ stopQueueObservation();
1300
+ await queueSnapshot;
1032
1301
  await termination;
1033
1302
  clearTimeout(boundTimer);
1034
1303
  cancelCapture();
@@ -1062,14 +1331,21 @@ export async function runBoundedExec(
1062
1331
  appendStderr(`${describeRamTempError(full)}\n`);
1063
1332
  }
1064
1333
  }
1065
- await releaseLease(code !== 0 || state.expiry !== null || state.poolCloseError !== null || lifecycleFailure !== null);
1334
+ await releaseLease(
1335
+ code !== 0 ||
1336
+ state.expiry !== null ||
1337
+ state.poolCloseError !== null ||
1338
+ lifecycleFailure !== null ||
1339
+ queueFailure !== null,
1340
+ );
1066
1341
 
1067
1342
  // Broken resource ownership, a lost native lifecycle, or leftovers that could not be retired fail even if a shell
1068
1343
  // branch later exited 0: its verdict is an ordinary operational failure, never a successful or silently unbounded run.
1069
- const operationalFailure = state.poolCloseError !== null || lifecycleFailure !== null || state.retireError !== null;
1344
+ const operationalFailure =
1345
+ state.poolCloseError !== null || lifecycleFailure !== null || state.retireError !== null || queueFailure !== null;
1070
1346
  await recordRun(code, state.expiry, Date.now() - startedAt, operationalFailure, seen, state.lockWaits);
1071
1347
  return {
1072
- success: !operationalFailure && state.expiry === null && code === 0,
1348
+ success: !operationalFailure && resultFailure === null && state.expiry === null && code === 0,
1073
1349
  terminalOutput: outputChunks.join(''),
1074
1350
  };
1075
1351
  }
@@ -383,11 +383,12 @@ export function stopProbe(file: string, pid: number): boolean {
383
383
  }
384
384
 
385
385
  /** A probe owns its group and output pipes; its deadline never waits for an unresponsive child to reap. */
386
- function runProbe(
386
+ export function runProbe(
387
387
  file: string,
388
388
  args: readonly string[],
389
389
  timeoutMs: number,
390
390
  signal: AbortSignal,
391
+ environment?: NodeJS.ProcessEnv,
391
392
  ): Promise<HangProbeResult> {
392
393
  // Keep the package's es2022-compatible event-completion convention.
393
394
  return new Promise((resolve) => {
@@ -395,7 +396,7 @@ function runProbe(
395
396
  resolve({ status: 'timed-out', detail: 'capture deadline already reached', stdout: '', stderr: '' });
396
397
  return;
397
398
  }
398
- const child = spawn(file, [...args], { detached: true, stdio: ['ignore', 'pipe', 'pipe'] });
399
+ const child = spawn(file, [...args], { detached: true, stdio: ['ignore', 'pipe', 'pipe'], env: environment });
399
400
  let stdout = '';
400
401
  let stderr = '';
401
402
  let bytes = 0;
@@ -1,4 +1,5 @@
1
- import type { RunnerKind } from './runner-sizing.js';
1
+ import { shellTokens } from '../../runtime-input-policy.js';
2
+ import { invocationSpan, type RunnerKind } from './runner-sizing.js';
2
3
  import { shellQuote } from './verdict.js';
3
4
 
4
5
  /**
@@ -11,9 +12,15 @@ export interface MbxPool {
11
12
  readonly priority: 'normal' | 'low';
12
13
  }
13
14
 
14
- /** The pool the environment names, or `null` where the project has no mbx and cowshed's own pools apply unchanged. */
15
+ /**
16
+ * The pool the environment names, or `null` where the project has no mbx and cowshed's own pools apply unchanged, or
17
+ * where this run is already inside an mbx reservation: `mbx reserve` hands its command `MBX_SCHED_DISABLE=1`, and nested
18
+ * non-Cargo work shares the enclosing grant instead of reserving twice (specs/cowshed/05_gateway.md, "Parallelism
19
+ * inside a job"). A second reservation would wait in the pool while its parent holds a grant, so under a full pool the
20
+ * nested leaf queued behind the very work it belongs to.
21
+ */
15
22
  export function mbxPool(environment: Readonly<NodeJS.ProcessEnv>): MbxPool | null {
16
- if (environment.MBX_SCHEDULER !== '1' || !environment.MBX_CACHE_DIR) {
23
+ if (environment.MBX_SCHEDULER !== '1' || !environment.MBX_CACHE_DIR || environment.MBX_SCHED_DISABLE === '1') {
17
24
  return null;
18
25
  }
19
26
  return { priority: environment.MBX_SCHEDULER_PRIORITY === 'normal' ? 'normal' : 'low' };
@@ -36,13 +43,83 @@ export function mbxLeaf(pool: MbxPool, kind: RunnerKind, want: number): MbxLeaf
36
43
  : { kind: 'reserve', cpus: Math.max(1, want), priority: pool.priority };
37
44
  }
38
45
 
46
+ /**
47
+ * Only a known scheduled execution may stop its idle clock at zero holds. Plain Cargo tests can run doctests;
48
+ * archive-file targets, direct binaries and other runners are not proven scheduled at spawn (11_shell.md,
49
+ * "Cargo test-runner boundaries"). The inferred extracted native nextest command is a supported Cargo invocation.
50
+ */
51
+ export function mbxIdleGated(leaf: MbxLeaf | null, command: string, environment: Readonly<NodeJS.ProcessEnv>): boolean {
52
+ if (leaf === null) return false;
53
+ if (leaf.kind === 'reserve') return true;
54
+ if (environment.MBX_SCHEDULER_TESTS !== '1') return false;
55
+ if (environment.MBX_SCHEDULER === '0' || environment.MBX_SCHED_DISABLE === '1' || environment.MBX_DISABLE === '1')
56
+ return false;
57
+ // invocationSpan shares the lifecycle mapper's conservative shell-syntax boundary.
58
+ if (/[<>\n\r`\\]/.test(command)) return false;
59
+ if (/[^\S \t]/.test(command)) return false;
60
+ const invocation = /cargo(?:\s+(?:--frozen|--locked|--offline))*\s+(nextest\s+run|test)(?=\s|$)/;
61
+ const found = invocation.exec(command);
62
+ if (found === null) return false;
63
+ const prefix = command.slice(0, found.index).trim();
64
+ if (prefix !== '' && !/^extracted="\$\([^"\n]+\)"\s*&&$/.test(prefix)) return false;
65
+ let remaining = command.slice(found.index);
66
+ const unquoted = remaining.replace(/"[^"]*"|'[^']*'/g, '');
67
+ if (/[$*?{}~[\]]/.test(unquoted)) return false;
68
+ // Only the inferred command's path templates are known expansions; arbitrary words could supply mbx overrides.
69
+ const paths = remaining.replace(/\$PWD(?=\/)/g, '');
70
+ if ((prefix === '' ? paths : paths.replace(/\$extracted(?=\/)/g, '')).includes('$')) return false;
71
+ for (;;) {
72
+ const leg = invocation.exec(remaining);
73
+ if (leg === null || leg.index !== 0) return false;
74
+ const { argsEnd } = invocationSpan(remaining, invocation);
75
+ const args: string[] = [];
76
+ for (const token of shellTokens(remaining.slice(0, argsEnd), [])) {
77
+ if (token.kind !== 'word') return false;
78
+ if (token.text === '--') break;
79
+ if (
80
+ token.text.startsWith('$') &&
81
+ !(
82
+ token.text.startsWith('$extracted/') &&
83
+ ['--binaries-metadata', '--cargo-metadata', '--target-dir-remap'].includes(args.at(-1) ?? '')
84
+ )
85
+ )
86
+ return false;
87
+ args.push(token.text);
88
+ }
89
+ if (
90
+ args.some((arg) =>
91
+ /^(?:--(?:config|directory|archive-file)(?:=|$)|--(?:doc|no-run|list|help)(?:=|$)|-C|-h$|\+)/.test(arg),
92
+ )
93
+ ) {
94
+ return false;
95
+ }
96
+ if (
97
+ leg[1] === 'test' &&
98
+ !args.some((arg) => /^--(?:lib|tests|bins|examples|benches|all-targets|test|bin|example|bench)(?:=|$)/.test(arg))
99
+ ) {
100
+ return false;
101
+ }
102
+ const suffix = remaining.slice(argsEnd).trim();
103
+ if (suffix === '') return true;
104
+ const separator = /^(?:&&|\|\||;)\s*/.exec(suffix);
105
+ if (separator === null) return false;
106
+ remaining = suffix.slice(separator[0].length);
107
+ }
108
+ }
109
+
110
+ /**
111
+ * mbx writes one byte per admission (g) and permit drop (r). Cargo descendants inherit this descriptor through
112
+ * MBX_GRANT_FD; reserve names it with --grant-fd and closes it on exec of the external command.
113
+ */
114
+ export const MBX_GRANT_FD = 5;
115
+
39
116
  /**
40
117
  * `command` as the shell runs it under its leaf's admission. `mbx` is the canonical environment's, whose realpath
41
- * cowshed's admission proved to be the host's pinned mbx.
118
+ * cowshed's admission proved to be the host's pinned mbx; it writes the grant on this descriptor itself.
42
119
  */
43
120
  export function withMbxLeaf(command: string, leaf: MbxLeaf): string {
44
121
  if (leaf.kind === 'cargo') {
45
122
  return command;
46
123
  }
47
- return `mbx reserve --cpus ${leaf.cpus} --priority ${leaf.priority} -- /bin/sh -c ${shellQuote(command)}`;
124
+ return `mbx reserve --grant-fd ${MBX_GRANT_FD} --cpus ${leaf.cpus} --priority ${leaf.priority} -- /bin/sh -c ${shellQuote(command)}`;
48
125
  }