@smoothbricks/nx-plugin 0.7.11 → 0.7.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (108) hide show
  1. package/README.md +87 -25
  2. package/dist/.build-inputs.json +25 -17
  3. package/dist/built-output-packages.d.ts +27 -0
  4. package/dist/built-output-packages.d.ts.map +1 -0
  5. package/dist/built-output-packages.js +97 -0
  6. package/dist/cross-check-policy.d.ts +13 -1
  7. package/dist/cross-check-policy.d.ts.map +1 -1
  8. package/dist/cross-check-policy.js +13 -1
  9. package/dist/executors/bounded-exec/admission-queue.d.ts +68 -0
  10. package/dist/executors/bounded-exec/admission-queue.d.ts.map +1 -0
  11. package/dist/executors/bounded-exec/admission-queue.js +131 -0
  12. package/dist/executors/bounded-exec/cpu-grant.d.ts +14 -0
  13. package/dist/executors/bounded-exec/cpu-grant.d.ts.map +1 -0
  14. package/dist/executors/bounded-exec/cpu-grant.js +18 -0
  15. package/dist/executors/bounded-exec/cpu-pool-thread.d.ts +5 -4
  16. package/dist/executors/bounded-exec/cpu-pool-thread.d.ts.map +1 -1
  17. package/dist/executors/bounded-exec/cpu-pool.d.ts +26 -24
  18. package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -1
  19. package/dist/executors/bounded-exec/cpu-pool.js +88 -72
  20. package/dist/executors/bounded-exec/executor.d.ts +5 -7
  21. package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
  22. package/dist/executors/bounded-exec/executor.js +69 -133
  23. package/dist/executors/bounded-exec/host-memory.d.ts +22 -0
  24. package/dist/executors/bounded-exec/host-memory.d.ts.map +1 -1
  25. package/dist/executors/bounded-exec/host-memory.darwin.d.ts.map +1 -1
  26. package/dist/executors/bounded-exec/host-memory.darwin.js +7 -1
  27. package/dist/executors/bounded-exec/host-memory.js +6 -0
  28. package/dist/executors/bounded-exec/host-memory.linux.d.ts +3 -1
  29. package/dist/executors/bounded-exec/host-memory.linux.d.ts.map +1 -1
  30. package/dist/executors/bounded-exec/host-memory.linux.js +43 -1
  31. package/dist/executors/bounded-exec/memory-retry.d.ts +39 -0
  32. package/dist/executors/bounded-exec/memory-retry.d.ts.map +1 -0
  33. package/dist/executors/bounded-exec/memory-retry.js +33 -0
  34. package/dist/executors/bounded-exec/parent-grant.d.ts +43 -0
  35. package/dist/executors/bounded-exec/parent-grant.d.ts.map +1 -0
  36. package/dist/executors/bounded-exec/parent-grant.js +37 -0
  37. package/dist/executors/bounded-exec/peak-charge.d.ts +17 -0
  38. package/dist/executors/bounded-exec/peak-charge.d.ts.map +1 -0
  39. package/dist/executors/bounded-exec/peak-charge.js +20 -0
  40. package/dist/executors/bounded-exec/pool-admission.d.ts +31 -7
  41. package/dist/executors/bounded-exec/pool-admission.d.ts.map +1 -1
  42. package/dist/executors/bounded-exec/pool-admission.js +44 -8
  43. package/dist/executors/bounded-exec/pool-environment.d.ts +25 -0
  44. package/dist/executors/bounded-exec/pool-environment.d.ts.map +1 -0
  45. package/dist/executors/bounded-exec/pool-environment.js +34 -0
  46. package/dist/executors/bounded-exec/run-cpu-pool.d.ts.map +1 -1
  47. package/dist/executors/bounded-exec/run-cpu-pool.js +2 -2
  48. package/dist/executors/bounded-exec/verdict.d.ts +5 -18
  49. package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
  50. package/dist/executors/bounded-exec/verdict.js +12 -16
  51. package/dist/index.d.ts +4 -1
  52. package/dist/index.d.ts.map +1 -1
  53. package/dist/index.js +62 -21
  54. package/dist/managed-files/ci-workflow.d.ts +29 -5
  55. package/dist/managed-files/ci-workflow.d.ts.map +1 -1
  56. package/dist/managed-files/ci-workflow.js +111 -16
  57. package/dist/managed-files/context.d.ts.map +1 -1
  58. package/dist/managed-files/context.js +1 -0
  59. package/dist/managed-files/files.d.ts +3 -1
  60. package/dist/managed-files/files.d.ts.map +1 -1
  61. package/dist/managed-files/files.js +2 -0
  62. package/dist/managed-files/publish-workflow.d.ts +6 -1
  63. package/dist/managed-files/publish-workflow.d.ts.map +1 -1
  64. package/dist/managed-files/publish-workflow.js +16 -11
  65. package/dist/owned-installation.d.ts.map +1 -1
  66. package/dist/owned-installation.js +48 -14
  67. package/dist/typecheck-route.d.ts +28 -0
  68. package/dist/typecheck-route.d.ts.map +1 -0
  69. package/dist/typecheck-route.js +109 -0
  70. package/dist/workspace-manifest.d.ts +50 -5
  71. package/dist/workspace-manifest.d.ts.map +1 -1
  72. package/executors.json +10 -0
  73. package/managed/raw/patches/nx@24.0.3.patch +16 -0
  74. package/managed/raw/patches/ttsc@0.34.1.patch +93 -8
  75. package/managed/raw/tooling/direnv/apple-developer.sh +8 -1
  76. package/managed/raw/tooling/direnv/compiler-variables.sh +1 -1
  77. package/managed/raw/tooling/direnv/github-actions-bootstrap.sh +4 -2
  78. package/managed/raw/tooling/direnv/secret-references.ts +19 -5
  79. package/managed/raw/tooling/git-hooks/pre-push.sh +20 -10
  80. package/managed/templates/github/actions/forgejo-artifact.cjs +9 -9
  81. package/managed/templates/github/actions/setup-devenv/action.yml +53 -0
  82. package/package.json +2 -2
  83. package/src/built-output-packages.ts +125 -0
  84. package/src/cross-check-policy.ts +14 -1
  85. package/src/executors/bounded-exec/admission-queue.ts +163 -0
  86. package/src/executors/bounded-exec/cpu-grant.ts +32 -0
  87. package/src/executors/bounded-exec/cpu-pool-thread.ts +5 -4
  88. package/src/executors/bounded-exec/cpu-pool.ts +123 -58
  89. package/src/executors/bounded-exec/executor.ts +93 -168
  90. package/src/executors/bounded-exec/host-memory.darwin.ts +7 -1
  91. package/src/executors/bounded-exec/host-memory.linux.ts +42 -2
  92. package/src/executors/bounded-exec/host-memory.ts +25 -0
  93. package/src/executors/bounded-exec/memory-retry.ts +63 -0
  94. package/src/executors/bounded-exec/parent-grant.ts +75 -0
  95. package/src/executors/bounded-exec/peak-charge.ts +25 -0
  96. package/src/executors/bounded-exec/pool-admission.ts +66 -11
  97. package/src/executors/bounded-exec/pool-environment.ts +45 -0
  98. package/src/executors/bounded-exec/run-cpu-pool.ts +5 -4
  99. package/src/executors/bounded-exec/verdict.ts +17 -25
  100. package/src/index.ts +70 -21
  101. package/src/managed-files/ci-workflow.ts +143 -16
  102. package/src/managed-files/context.ts +1 -0
  103. package/src/managed-files/files.ts +5 -0
  104. package/src/managed-files/publish-workflow.ts +26 -10
  105. package/src/owned-installation.ts +63 -14
  106. package/src/typecheck-route.ts +64 -0
  107. package/src/workspace-manifest.ts +56 -5
  108. package/test-timings.json +141 -137
@@ -13,31 +13,13 @@ import {
13
13
  processGroupHasMembers,
14
14
  signalProcessGroup,
15
15
  } from '../../process-group.js';
16
- import {
17
- admissionWaitMs,
18
- type BoundEvent,
19
- type BoundKind,
20
- type Bounds,
21
- earliestBound,
22
- NO_ADMISSION_WAIT,
23
- stepAdmissionWait,
24
- stepBounds,
25
- UNARMED,
26
- } from './bounds.js';
27
- import {
28
- CPU_POOL_ENV,
29
- type CpuPool,
30
- createCpuPool,
31
- type InheritedCpuPool,
32
- readInheritedCpuPool,
33
- type TokenGrant,
34
- takeParentCpuTokens,
35
- } from './cpu-pool.js';
16
+ import { AdmissionObservation, type QueueHost } from './admission-queue.js';
17
+ import { type BoundEvent, type BoundKind, type Bounds, earliestBound, stepBounds, UNARMED } from './bounds.js';
18
+ import { type CpuPool, createCpuPool } from './cpu-pool.js';
36
19
  import { GATEWAY_SOCKET } from './gateway-lease.js';
37
20
  import type { HangDiagnostics } from './hang-diagnostics.js';
38
21
  import { captureHangDiagnostics, HANG_CAPTURE_MS } from './hang-diagnostics.js';
39
22
  import { type LeafPeak, PEAK_FD, peakWaiter, readWaiterReport, underWaiter } from './leaf-peak.js';
40
- import type { CpuWaitRow } from './lease-wait.js';
41
23
  import type { LockWait } from './lock-wait.js';
42
24
  import {
43
25
  askedCores,
@@ -48,8 +30,8 @@ import {
48
30
  withCountedReservation,
49
31
  withMbxLeaf,
50
32
  } from './mbx-pool.js';
51
- import { MBX_QUEUE_POLL_MS, type MbxQueuedRequest, snapshotMbxQueue } from './mbx-queue.js';
52
33
  import { describeUnserved, RESERVATIONS_ENV, type Reservation, reserveCores, runEndpoint } from './mbx-reservations.js';
34
+ import { killedFromOutside, type RunOutcome, runUntilNotKilledForMemory } from './memory-retry.js';
53
35
  import { createNoSpaceWatch } from './no-space.js';
54
36
  import {
55
37
  COWSHED_EXECUTABLE_ENV,
@@ -60,6 +42,9 @@ import {
60
42
  type SubtreeAdmission,
61
43
  type SubtreeOwner,
62
44
  } from './owned-subtree.js';
45
+ import { askParentPool, describeGrant } from './parent-grant.js';
46
+ import { type LastPeak, readLastPeak } from './peak-charge.js';
47
+ import { CPU_POOL_ENV, type InheritedCpuPool, readInheritedCpuPool } from './pool-environment.js';
63
48
  import { createRunProgress, withProgressReport, withTestResultReport } from './progress.js';
64
49
  import {
65
50
  createHostCommands,
@@ -79,12 +64,10 @@ import { checkRuntimePin, describeRuntimePinRefusal } from './runtime-pin.js';
79
64
  import type { BoundedExecOptions } from './schema.js';
80
65
  import { createTestResultWriter, type TestResultWriter } from './test-result-writer.js';
81
66
  import {
82
- type AdmissionQueueWait,
83
67
  type BoundedExecRecord,
84
68
  type BoundedExecTask,
85
69
  BUN_TEST,
86
70
  clearTaskDirectory,
87
- describeAdmissionQueue,
88
71
  describeNoSpace,
89
72
  type ExpiredBound,
90
73
  JUNIT_FILE,
@@ -93,7 +76,6 @@ import {
93
76
  judgeRun,
94
77
  type NoSpaceSighting,
95
78
  readJunitReport,
96
- readLastPeak,
97
79
  taskDirectory,
98
80
  withJunitReport,
99
81
  writeRecord,
@@ -131,15 +113,14 @@ export interface NxTaskContext {
131
113
  taskGraph?: { tasks: Record<string, { hash?: string }> };
132
114
  }
133
115
 
134
- export interface BoundedExecResult {
135
- success: boolean;
136
- terminalOutput: string;
137
- }
116
+ export type BoundedExecResult = RunOutcome;
138
117
 
139
118
  interface RunState {
140
119
  settled: boolean;
141
120
  expiry: ExpiredBound | null;
142
121
  forceKillNeeded: boolean;
122
+ /** This executor sent the command's process group a signal: a SIGKILL it died of is not from outside. */
123
+ sentSignal: boolean;
143
124
  /** The sub-pool's close error when it did not close cleanly: the run fails. */
144
125
  poolCloseError: string | null;
145
126
  /** Why the command's subtree could not be retired, when the supervisor's client failed: the run fails. */
@@ -150,8 +131,6 @@ interface RunState {
150
131
  lockWaits: readonly LockWait[];
151
132
  }
152
133
 
153
- type GrantedTokens = Extract<TokenGrant, { granted: true }>;
154
-
155
134
  export interface ProcessTreeKiller {
156
135
  kill(pid: number, signal: NodeJS.Signals): Promise<void>;
157
136
  }
@@ -311,12 +290,45 @@ export async function runBoundedExec(
311
290
  inherited: InheritedCpuPool | null = null,
312
291
  captureDiagnostics: typeof captureHangDiagnostics = captureHangDiagnostics,
313
292
  owner: SubtreeOwner | null = null,
314
- queueHost?: Parameters<typeof snapshotMbxQueue>[3],
293
+ queueHost?: QueueHost,
315
294
  ): Promise<BoundedExecResult> {
316
- const cwd = resolveCwd(options.cwd, context.root);
317
295
  const directory = context.record?.directory ?? null;
318
- // The task's last recorded peak is its charge in its parent's pool, read before the clear below removes it.
296
+ // The task's last recorded peak, and the tokens it was lent when the peak was measured, are its charge in its parent's
297
+ // pool, read once before a run's clear removes them: a command run again after a kill is charged by the record the
298
+ // killed run replaced, not by the peak it reached before it was killed.
319
299
  const lastPeak = directory !== null && inherited !== null ? await readLastPeak(directory) : null;
300
+ return runUntilNotKilledForMemory(
301
+ (alone) =>
302
+ runCommand(options, context, killer, tempVolume, inherited, captureDiagnostics, owner, queueHost, {
303
+ lastPeak,
304
+ alone,
305
+ }),
306
+ inherited !== null,
307
+ context.record?.task.id,
308
+ );
309
+ }
310
+
311
+ /** What a run of the command is told that the run before it was not: the task's charge, and whether it runs alone. */
312
+ interface Attempt {
313
+ readonly lastPeak: LastPeak | null;
314
+ readonly alone: boolean;
315
+ }
316
+
317
+ /** One run of the command, from the pool's grant to its record. */
318
+ async function runCommand(
319
+ options: BoundedExecOptions,
320
+ context: BoundedExecContext,
321
+ killer: ProcessTreeKiller,
322
+ tempVolume: TempVolume | null,
323
+ inherited: InheritedCpuPool | null,
324
+ captureDiagnostics: typeof captureHangDiagnostics,
325
+ owner: SubtreeOwner | null,
326
+ queueHost: QueueHost | undefined,
327
+ attempt: Attempt,
328
+ ): Promise<BoundedExecResult> {
329
+ const cwd = resolveCwd(options.cwd, context.root);
330
+ const directory = context.record?.directory ?? null;
331
+ const { lastPeak } = attempt;
320
332
  // A previous run's verdict and report go first, so neither can be read as this run's.
321
333
  if (directory !== null) {
322
334
  await clearTaskDirectory(directory);
@@ -336,6 +348,7 @@ export async function runBoundedExec(
336
348
  settled: false,
337
349
  expiry: null,
338
350
  forceKillNeeded: false,
351
+ sentSignal: false,
339
352
  poolCloseError: null,
340
353
  retireError: null,
341
354
  noSpace: null,
@@ -408,8 +421,10 @@ export async function runBoundedExec(
408
421
  `bounded-exec: ${lastPeak.error}; it is this task's memory charge, so the command does not run\n`,
409
422
  );
410
423
  }
411
- // A task with no recorded peak is charged nothing (null) until it has one.
424
+ // A task with no recorded peak is charged nothing (null) until it has one. A peak is charged in proportion to the
425
+ // tokens the run is lent when that is more than the run that measured it was lent.
412
426
  const charge = lastPeak?.ok === true ? lastPeak.peakBytes : null;
427
+ const chargeParallelism = lastPeak?.ok === true ? lastPeak.parallelism : null;
413
428
 
414
429
  let lease: RamTempLease | null = null;
415
430
  let ordinaryTemp: string | null = null;
@@ -553,20 +568,15 @@ export async function runBoundedExec(
553
568
  const asked = Date.now();
554
569
  // A pool that answered and refused is a live budget, and running beside it would count the run twice: the run fails
555
570
  // instead. A pool whose endpoint is gone (its holder exited) lends nothing, and the run goes on as asked.
556
- let grant: GrantedTokens | null = null;
557
- let parentRefusal: string | null = null;
558
- let parentWait: CpuWaitRow | undefined;
559
- if (inherited !== null) {
560
- const fromParent = await takeParentCpuTokens(inherited, demand.want, charge);
561
- if (fromParent.granted) {
562
- grant = fromParent;
563
- } else if (fromParent.cause === 'absent') {
564
- appendStderr(`cowshed: cpu-pool parent pool absent: ${fromParent.reason}; running the command as asked\n`);
565
- } else {
566
- parentRefusal = fromParent.reason;
567
- parentWait = fromParent.wait;
568
- }
569
- }
571
+ const parent = await askParentPool(inherited, {
572
+ want: demand.want,
573
+ charge,
574
+ parallelism: chargeParallelism,
575
+ alone: attempt.alone,
576
+ task: context.record?.task.id,
577
+ });
578
+ for (const said of parent.said) appendStderr(said);
579
+ const { grant, refusal: parentRefusal, wait: parentWait } = parent;
570
580
  const waitedMs = Date.now() - asked;
571
581
  let cpu: BoundedExecRecord['cpu'];
572
582
  let sized: { command: string; env: Record<string, string> } = { command: requested, env: {} };
@@ -577,13 +587,8 @@ export async function runBoundedExec(
577
587
  if (grant !== null) {
578
588
  sized = sizeRunner(demand, requested, grant.tokens);
579
589
  cpu = { tokens: grant.tokens, want: demand.want, waitedMs };
580
- appendStderr(
581
- `cowshed: cpu-pool wait done elapsed=${waitedMs}ms status=ok tokens=${grant.tokens}/${demand.want} runner=${demand.kind} memory=${charge ?? 'none'}\n`,
582
- );
583
- if (grant.oversized !== null) {
584
- appendStderr(
585
- `cowshed: cpu-pool granted while nothing else holds memory: this task's last peak of ${charge} bytes alone exceeds the run's ${grant.oversized.memory} bytes\n`,
586
- );
590
+ for (const said of describeGrant(grant, { waitedMs, want: demand.want, runner: demand.kind, charge })) {
591
+ appendStderr(said);
587
592
  }
588
593
  } else if (leaf?.kind === 'reserve') {
589
594
  sized = { command: sizeRunnerToGrant(leaf.demand, requested), env: {} };
@@ -611,33 +616,15 @@ export async function runBoundedExec(
611
616
  })();
612
617
  return cpuReleased;
613
618
  };
614
- let mbxAdmitted = false;
615
- let admissionWait = NO_ADMISSION_WAIT;
616
- let lastQueueRequests: readonly MbxQueuedRequest[] = [];
617
- /**
618
- * Why the last admission snapshot failed, while the one after it has not succeeded; null otherwise. A snapshot only
619
- * decides whether waiting on mbx admission is credited against the idle bound, so a failed one withholds that credit
620
- * and ends nothing: the bounds decide, and an idle expiry names it.
621
- */
622
- let unconfirmedAdmission: string | null = null;
623
- /** When the last successful snapshot completed (`Date.now()`): credit runs to it and no further. */
624
- let confirmedAdmissionAt: number | null = null;
625
- // A reserved leaf's own request on its run's endpoint is its admission wait; a Cargo leaf's comes from mbx's report.
626
- const queueWait = (): AdmissionQueueWait | null =>
627
- mbxAdmitted || (lastQueueRequests.length === 0 && reservation === null)
628
- ? null
629
- : {
630
- waitedMs: admissionWaitMs(admissionWait, performance.now()),
631
- requests: lastQueueRequests,
632
- };
633
- const queueAbort = new AbortController();
634
- let queueTimer: NodeJS.Timeout | undefined;
635
- let queueSnapshot: Promise<void> | null = null;
636
- const stopQueueObservation = (): void => {
637
- admissionWait = stepAdmissionWait(admissionWait, { kind: 'stop', at: performance.now() });
638
- clearInterval(queueTimer);
639
- queueAbort.abort();
640
- };
619
+ const queue = new AdmissionObservation({
620
+ over: () => state.settled || state.expiry !== null,
621
+ pid: () => child.pid,
622
+ environment: () => mergeEnv(lease?.directory ?? ordinaryTemp, env),
623
+ tell,
624
+ step: (event) => stepTo(event),
625
+ testing: () => isTesting(),
626
+ queueHost,
627
+ });
641
628
  // Records the run's verdict when it has a task to record as. An `expiry` is set only when a bound ended the run.
642
629
  // Observation and ownership failures remain operational even after valid queue evidence; cancellation alone does not.
643
630
  const recordRun = async (
@@ -671,7 +658,7 @@ export async function runBoundedExec(
671
658
  appendStderr(`bounded-exec: ${error}\n`);
672
659
  }
673
660
  });
674
- const admissionQueue = queueWait();
661
+ const admissionQueue = queue.queueWait(reservation !== null);
675
662
  const verdict =
676
663
  operational || resultFailure !== null
677
664
  ? judgeOperationalFailure(exitCode, report)
@@ -960,7 +947,7 @@ export async function runBoundedExec(
960
947
  try {
961
948
  // Drain an older poll, then take a fresh bounded snapshot before idle can end queued work. Total and fixture
962
949
  // ceilings remain armed while this observation runs.
963
- await refreshQueue(true);
950
+ await queue.refresh(true);
964
951
  } finally {
965
952
  checkingIdle = false;
966
953
  }
@@ -990,74 +977,6 @@ export async function runBoundedExec(
990
977
  armBoundTimer();
991
978
  };
992
979
 
993
- let lastQueueDiagnostic = '';
994
- let lastQueueDiagnosticAt: number | null = null;
995
- let lastSnapshotDiagnostics = '';
996
- const unconfirmAdmission = (detail: string): void => {
997
- if (state.settled || state.expiry !== null || queueAbort.signal.aborted) return;
998
- if (unconfirmedAdmission !== detail) {
999
- tell(
1000
- `bounded-exec: mbx admission snapshot failed: ${detail}; admission is unconfirmed, so this wait earns no idle credit until a snapshot succeeds\n`,
1001
- );
1002
- }
1003
- unconfirmedAdmission = detail;
1004
- lastQueueRequests = [];
1005
- admissionWait = stepAdmissionWait(admissionWait, { kind: 'snapshot', at: performance.now(), waiting: false });
1006
- // The wait since the last confirmed snapshot is unconfirmed: the idle bound resumes from that snapshot.
1007
- stepTo({ kind: 'admission-queue', at: confirmedAdmissionAt ?? Date.now(), waiting: false, testing: isTesting() });
1008
- };
1009
- const refreshQueue = async (fresh: boolean): Promise<void> => {
1010
- if (fresh && queueSnapshot !== null) await queueSnapshot;
1011
- if (state.settled || state.expiry !== null || queueAbort.signal.aborted || child.pid === undefined) return;
1012
- const pid = child.pid;
1013
- queueSnapshot ??= (async () => {
1014
- const snapshot = await snapshotMbxQueue(
1015
- pid,
1016
- mergeEnv(lease?.directory ?? ordinaryTemp, env),
1017
- queueAbort.signal,
1018
- queueHost,
1019
- );
1020
- if (state.settled || state.expiry !== null || queueAbort.signal.aborted) return;
1021
- if (!snapshot.ok) {
1022
- unconfirmAdmission(snapshot.detail);
1023
- return;
1024
- }
1025
- const diagnostics = snapshot.diagnostics?.join('; ') ?? '';
1026
- if (diagnostics !== '' && diagnostics !== lastSnapshotDiagnostics) tell(`bounded-exec: ${diagnostics}\n`);
1027
- lastSnapshotDiagnostics = diagnostics;
1028
- const waiting = snapshot.requests.length > 0;
1029
- const observedAt = Date.now();
1030
- const observedWaitAt = performance.now();
1031
- admissionWait = stepAdmissionWait(admissionWait, { kind: 'snapshot', at: observedWaitAt, waiting });
1032
- unconfirmedAdmission = null;
1033
- confirmedAdmissionAt = observedAt;
1034
- if (waiting) {
1035
- lastQueueRequests = snapshot.requests;
1036
- const positions = JSON.stringify(snapshot.requests);
1037
- if (
1038
- positions !== lastQueueDiagnostic ||
1039
- lastQueueDiagnosticAt === null ||
1040
- Date.now() - lastQueueDiagnosticAt >= HANG_CAPTURE_MS
1041
- ) {
1042
- const waitedMs = admissionWaitMs(admissionWait, observedWaitAt);
1043
- tell(`cowshed: mbx ${describeAdmissionQueue({ waitedMs, requests: snapshot.requests })}\n`);
1044
- lastQueueDiagnostic = positions;
1045
- lastQueueDiagnosticAt = Date.now();
1046
- }
1047
- } else {
1048
- lastQueueDiagnostic = '';
1049
- }
1050
- stepTo({ kind: 'admission-queue', at: observedAt, waiting, testing: isTesting() });
1051
- })()
1052
- .catch((error: unknown) => {
1053
- unconfirmAdmission(error instanceof Error ? error.message : String(error));
1054
- })
1055
- .finally(() => {
1056
- queueSnapshot = null;
1057
- });
1058
- await queueSnapshot;
1059
- };
1060
-
1061
980
  // Only the CHILD's own output is read for progress. The diagnostics below go
1062
981
  // through appendStderr as well, and re-arming on those would let a teardown
1063
982
  // message extend the very bound that just fired.
@@ -1131,6 +1050,7 @@ export async function runBoundedExec(
1131
1050
  return;
1132
1051
  }
1133
1052
  try {
1053
+ state.sentSignal = true;
1134
1054
  await killer.kill(pid, 'SIGTERM');
1135
1055
  } catch (error) {
1136
1056
  const reason = error instanceof Error ? error.message : String(error);
@@ -1148,9 +1068,9 @@ export async function runBoundedExec(
1148
1068
  clearTimeout(boundTimer);
1149
1069
  // Preserve explicit live membership before cancellation removes the queued process. A failed bounded observation
1150
1070
  // leaves the admission unconfirmed, never a manufactured queue or admission.
1151
- signalCleanup = (leaf?.kind !== 'cargo' ? Promise.resolve() : refreshQueue(false))
1071
+ signalCleanup = (leaf?.kind !== 'cargo' ? Promise.resolve() : queue.refresh(false))
1152
1072
  .finally(() => {
1153
- stopQueueObservation();
1073
+ queue.stop();
1154
1074
  return killChildTree();
1155
1075
  })
1156
1076
  .finally(releaseCpu)
@@ -1192,12 +1112,12 @@ export async function runBoundedExec(
1192
1112
  child.once('error', (error) => {
1193
1113
  appendStderr(`${error.message}\n`);
1194
1114
  state.settled = true;
1195
- stopQueueObservation();
1115
+ queue.stop();
1196
1116
  resolve({ code: 1, signal: null });
1197
1117
  });
1198
1118
  child.once(hooked === null ? 'exit' : 'close', (code: number | null, signal: NodeJS.Signals | null) => {
1199
1119
  state.settled = true;
1200
- stopQueueObservation();
1120
+ queue.stop();
1201
1121
  resolve({ code, signal });
1202
1122
  });
1203
1123
  });
@@ -1221,9 +1141,9 @@ export async function runBoundedExec(
1221
1141
  ? `still running: ${bound.tests.join(', ')}`
1222
1142
  : 'the runner named no test still running'
1223
1143
  } (cwd=${cwd}): ${command}\n${
1224
- unconfirmedAdmission === null
1144
+ queue.unconfirmed === null
1225
1145
  ? ''
1226
- : `The last mbx admission snapshot failed (${unconfirmedAdmission}), so the wait it could not confirm earned no idle credit\n`
1146
+ : `The last mbx admission snapshot failed (${queue.unconfirmed}), so the wait it could not confirm earned no idle credit\n`
1227
1147
  }`
1228
1148
  : bound.kind === 'fixture'
1229
1149
  ? `\nFixture command timed out after ${elapsedMs}ms (fixtureTimeoutMs=${bound.limitMs}, cwd=${cwd}): ${segment ?? command}\n`
@@ -1280,6 +1200,7 @@ export async function runBoundedExec(
1280
1200
  // cannot end is said, and a failure of the probe or of a signal is shown.
1281
1201
  const forceKill = async (pid: number): Promise<void> => {
1282
1202
  try {
1203
+ state.sentSignal = true;
1283
1204
  if (await killProcessGroup(pid, DEADLINE_PATIENCE, (group) => killer.kill(group, 'SIGKILL'))) {
1284
1205
  tell(`${describeMembersLeft(pid)}\n`);
1285
1206
  }
@@ -1359,7 +1280,7 @@ export async function runBoundedExec(
1359
1280
  channel.end();
1360
1281
  return;
1361
1282
  }
1362
- admissionWait = stepAdmissionWait(admissionWait, { kind: 'snapshot', at: performance.now(), waiting: true });
1283
+ queue.waiting();
1363
1284
  tell(`cowshed: mbx reserve wait start cpus=${cpus} priority=${reservedLeaf.priority}\n`);
1364
1285
  const held = reserveCores(endpoint, cpus, reservedLeaf.priority);
1365
1286
  reservation = held;
@@ -1371,7 +1292,7 @@ export async function runBoundedExec(
1371
1292
  }
1372
1293
  // A run that ended while it waited gives the grant back with its CPU (`releaseCpu`).
1373
1294
  if (state.settled || state.expiry !== null) return;
1374
- mbxAdmitted = true;
1295
+ queue.admitted = true;
1375
1296
  tell(`cowshed: mbx reserve granted elapsed=${Date.now() - startedAt}ms priority=${reservedLeaf.priority}\n`);
1376
1297
  channel.end(`${granted.cpus}\n`);
1377
1298
  stepTo({ kind: 'granted', at: Date.now(), testing: isTesting(), fixture: !fixtureDone });
@@ -1390,7 +1311,7 @@ export async function runBoundedExec(
1390
1311
  for (const byte of chunk) {
1391
1312
  if (byte === 103) {
1392
1313
  if (!said) {
1393
- mbxAdmitted = true;
1314
+ queue.admitted = true;
1394
1315
  said = true;
1395
1316
  tell(`cowshed: mbx cargo granted elapsed=${Date.now() - startedAt}ms\n`);
1396
1317
  }
@@ -1404,10 +1325,7 @@ export async function runBoundedExec(
1404
1325
  }
1405
1326
  });
1406
1327
  // Only mbx's activity report says whether a Cargo leaf's compiler or test waits in the pool.
1407
- queueTimer = setInterval(() => {
1408
- void refreshQueue(false);
1409
- }, MBX_QUEUE_POLL_MS);
1410
- void refreshQueue(false);
1328
+ queue.poll();
1411
1329
  }
1412
1330
  }
1413
1331
 
@@ -1432,8 +1350,8 @@ export async function runBoundedExec(
1432
1350
  await signalCleanup;
1433
1351
  return { success: false, terminalOutput: outputChunks.join('') };
1434
1352
  }
1435
- stopQueueObservation();
1436
- await queueSnapshot;
1353
+ queue.stop();
1354
+ await queue.idle();
1437
1355
  await termination;
1438
1356
  clearTimeout(boundTimer);
1439
1357
  cancelCapture();
@@ -1488,9 +1406,16 @@ export async function runBoundedExec(
1488
1406
  // branch later exited 0: its verdict is an ordinary operational failure, never a successful or silently unbounded run.
1489
1407
  const operationalFailure = state.poolCloseError !== null || lifecycleFailure !== null || state.retireError !== null;
1490
1408
  await recordRun(code, state.expiry, Date.now() - startedAt, operationalFailure, seen, state.lockWaits, peakBytes);
1409
+ const killedForMemory = killedFromOutside({
1410
+ sentSignal: state.sentSignal,
1411
+ bounded: state.expiry !== null,
1412
+ exitSignal,
1413
+ code,
1414
+ });
1491
1415
  return {
1492
1416
  success: !operationalFailure && resultFailure === null && state.expiry === null && code === 0,
1493
1417
  terminalOutput: outputChunks.join(''),
1418
+ ...(killedForMemory ? { killedForMemory: true } : {}),
1494
1419
  };
1495
1420
  }
1496
1421
 
@@ -132,4 +132,10 @@ async function treeResidentBytes(root: number): Promise<number> {
132
132
  return total;
133
133
  }
134
134
 
135
- export const darwinMemory: HostMemory = { readRunMemory, peakBytes: (maxRss) => maxRss, treeResidentBytes };
135
+ export const darwinMemory: HostMemory = {
136
+ readRunMemory,
137
+ peakBytes: (maxRss) => maxRss,
138
+ treeResidentBytes,
139
+ // macOS has no cgroup, and its anonymous and shared pages are not read from a file the pool may open.
140
+ liveUsage: () => ({ ok: false, reason: 'macOS keeps no anonymous and shared memory of a run to read' }),
141
+ };
@@ -1,7 +1,8 @@
1
+ import { readFileSync } from 'node:fs';
1
2
  import { readFile } from 'node:fs/promises';
2
3
  import { posix } from 'node:path';
3
4
  import { childPids } from '../../process-children.linux.js';
4
- import type { HostMemory, RunMemory, RunMemoryError, RunMemoryReading } from './host-memory.js';
5
+ import type { HostMemory, LiveUsageReading, RunMemory, RunMemoryError, RunMemoryReading } from './host-memory.js';
5
6
 
6
7
  /**
7
8
  * Linux's memory mechanisms. The run's memory is the least of the host's `MemTotal` and every finite `memory.max` of
@@ -103,6 +104,40 @@ async function readText(path: string): Promise<FileRead> {
103
104
  }
104
105
  }
105
106
 
107
+ /** The memory a bound holds live: `memory.stat`'s `anon` and `shmem` beside a cgroup's `memory.max`, meminfo's `AnonPages` and `Shmem` for the host's. */
108
+ export function linuxLiveUsage(read: (path: string) => FileRead, bound: string): LiveUsageReading {
109
+ const cgroupLimit = '/memory.max';
110
+ if (bound.endsWith(cgroupLimit)) {
111
+ const file = `${bound.slice(0, -cgroupLimit.length)}/memory.stat`;
112
+ const stat = read(file);
113
+ if (!stat.ok) return { ok: false, reason: `${file}: ${stat.code}: ${stat.message}` };
114
+ const [anon, shmem] = [/^anon (\d+)$/m.exec(stat.text)?.[1], /^shmem (\d+)$/m.exec(stat.text)?.[1]];
115
+ return anon === undefined || shmem === undefined
116
+ ? { ok: false, reason: `${file} names no anon and shmem in bytes` }
117
+ : { ok: true, usage: { anon: Number(anon), shmem: Number(shmem), source: file } };
118
+ }
119
+ if (bound === `${MEMINFO} MemTotal`) {
120
+ const info = read(MEMINFO);
121
+ if (!info.ok) return { ok: false, reason: `${MEMINFO}: ${info.code}: ${info.message}` };
122
+ const [anon, shmem] = [
123
+ /^AnonPages:\s+(\d+) kB$/m.exec(info.text)?.[1],
124
+ /^Shmem:\s+(\d+) kB$/m.exec(info.text)?.[1],
125
+ ];
126
+ return anon === undefined || shmem === undefined
127
+ ? { ok: false, reason: `${MEMINFO} names no AnonPages and Shmem in kB` }
128
+ : { ok: true, usage: { anon: Number(anon) * 1024, shmem: Number(shmem) * 1024, source: MEMINFO } };
129
+ }
130
+ return { ok: false, reason: `${bound} is no cgroup memory.max or meminfo bound` };
131
+ }
132
+
133
+ function readTextSync(path: string): FileRead {
134
+ try {
135
+ return { ok: true, text: readFileSync(path, 'utf8') };
136
+ } catch (error) {
137
+ const code = error instanceof Error && 'code' in error && typeof error.code === 'string' ? error.code : 'EIO';
138
+ return { ok: false, code, message: error instanceof Error ? error.message : String(error) };
139
+ }
140
+ }
106
141
  /** Bun before 1.4.0 reports Linux's `ru_maxrss` as the kernel does, in KiB; from 1.4.0 it reports bytes. */
107
142
  function peakBytes(maxRss: number, bunVersion: string): number {
108
143
  const version = /^(\d+)\.(\d+)\./.exec(bunVersion);
@@ -144,4 +179,9 @@ async function residentBytes(pid: number): Promise<number> {
144
179
  return kib === undefined ? 0 : Number(kib) * 1024;
145
180
  }
146
181
 
147
- export const linuxMemory: HostMemory = { readRunMemory: () => linuxRunMemory(readText), peakBytes, treeResidentBytes };
182
+ export const linuxMemory: HostMemory = {
183
+ readRunMemory: () => linuxRunMemory(readText),
184
+ peakBytes,
185
+ treeResidentBytes,
186
+ liveUsage: (bound) => linuxLiveUsage(readTextSync, bound),
187
+ };
@@ -21,8 +21,26 @@ export interface HostMemory {
21
21
  * listing and its read adds nothing, and a root that has exited holds nothing.
22
22
  */
23
23
  treeResidentBytes(root: number): Promise<number>;
24
+ /**
25
+ * What the run's bound holds that the kernel cannot take back, read now: its anonymous and shared memory. `bound` is
26
+ * the `source` of the run's memory (`RunMemory`): the cgroup's `memory.max`, whose `memory.stat` is read, or the
27
+ * host's `MemTotal`, whose `AnonPages` and `Shmem` are. The pool says it on every grant, so what the leaves are not
28
+ * charged for (the baseline, a tmpfs) and what a suite holds beyond its record are measured, not guessed.
29
+ */
30
+ liveUsage(bound: string): LiveUsageReading;
31
+ }
32
+
33
+ /** What a bound holds live, in bytes, and the file it was read from. */
34
+ export interface LiveUsage {
35
+ readonly anon: number;
36
+ readonly shmem: number;
37
+ readonly source: string;
24
38
  }
25
39
 
40
+ export type LiveUsageReading =
41
+ | { readonly ok: true; readonly usage: LiveUsage }
42
+ | { readonly ok: false; readonly reason: string };
43
+
26
44
  /** The run's memory in bytes, and where it was read: the file or sysctl that holds it. */
27
45
  export interface RunMemory {
28
46
  readonly bytes: number;
@@ -75,3 +93,10 @@ export function readRunMemory(): Promise<RunMemoryReading> {
75
93
  }
76
94
  return HOST_MEMORY.readRunMemory();
77
95
  }
96
+
97
+ /** The run's bound live on this platform, or why it cannot be read: a pool says it, never leaves it out. */
98
+ export function readLiveUsage(bound: string): LiveUsageReading {
99
+ return HOST_MEMORY === null
100
+ ? { ok: false, reason: `${process.platform} has no reader of live memory` }
101
+ : HOST_MEMORY.liveUsage(bound);
102
+ }
@@ -0,0 +1,63 @@
1
+ /**
2
+ * A run again after the kernel killed the command for memory (specs/cowshed/05_gateway.md, "Host scheduling"). Only
3
+ * when a pool lent the run its tokens and a SIGKILL that no bound of this executor and no signal it sent caused ended
4
+ * it: the command runs once more, alone in its run's pool, so what it was killed beside is not there to be killed with
5
+ * it. This decides whether a command runs a second time and how its two outputs join, never what it runs: the key
6
+ * leaves it out (README, "Bounded exec key").
7
+ */
8
+
9
+ /** How one run of the command ended. */
10
+ export interface RunOutcome {
11
+ success: boolean;
12
+ terminalOutput: string;
13
+ /**
14
+ * The command died of a SIGKILL that no bound of this executor and no signal it sent caused: the kernel's OOM killer,
15
+ * which sends it to a leaf in a run's cgroup. Set on the last run of the command only when it was killed again.
16
+ */
17
+ killedForMemory?: boolean;
18
+ }
19
+
20
+ /** What a run read of how its command ended and what this executor did to it. */
21
+ export interface Ending {
22
+ /** This executor sent the command's process group a signal. */
23
+ readonly sentSignal: boolean;
24
+ /** A bound of this executor ended the run. */
25
+ readonly bounded: boolean;
26
+ readonly exitSignal: NodeJS.Signals | null;
27
+ /** The exit status, or the status a signal folds to. */
28
+ readonly code: number;
29
+ }
30
+
31
+ /**
32
+ * A SIGKILL this executor did not send, to a command that no bound of this executor ended, came from outside it. In a
33
+ * run's cgroup the one that sends it to a leaf is the kernel's OOM killer. The exit code alone says nothing of who sent
34
+ * it: a bound's own force-kill ends in 137 too, and is why `sentSignal` is read first.
35
+ */
36
+ export function killedFromOutside(ending: Ending): boolean {
37
+ return !ending.sentSignal && !ending.bounded && (ending.exitSignal === 'SIGKILL' || ending.code === 137);
38
+ }
39
+
40
+ /**
41
+ * Run the command; when the kernel killed it for memory and a pool lent it tokens, run it once more alone, and join the
42
+ * two runs' output with a note naming the task.
43
+ */
44
+ export async function runUntilNotKilledForMemory(
45
+ run: (alone: boolean) => Promise<RunOutcome>,
46
+ pooled: boolean,
47
+ task: string | undefined,
48
+ ): Promise<RunOutcome> {
49
+ const first = await run(false);
50
+ if (first.killedForMemory !== true || !pooled) {
51
+ return first;
52
+ }
53
+ const note = `cowshed: the kernel killed the command for memory (a SIGKILL this executor did not send${
54
+ task === undefined ? '' : `: ${task}`
55
+ }); running it once more, alone in its run's pool\n`;
56
+ process.stderr.write(note);
57
+ const again = await run(true);
58
+ return {
59
+ success: again.success,
60
+ terminalOutput: `${first.terminalOutput}${note}${again.terminalOutput}`,
61
+ ...(again.killedForMemory === true ? { killedForMemory: true } : {}),
62
+ };
63
+ }