@smoothbricks/nx-plugin 0.7.12 → 0.7.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (89) hide show
  1. package/README.md +91 -27
  2. package/dist/.build-inputs.json +20 -12
  3. package/dist/built-output-packages.d.ts +27 -0
  4. package/dist/built-output-packages.d.ts.map +1 -0
  5. package/dist/built-output-packages.js +97 -0
  6. package/dist/cross-check-policy.d.ts +13 -1
  7. package/dist/cross-check-policy.d.ts.map +1 -1
  8. package/dist/cross-check-policy.js +13 -1
  9. package/dist/executors/bounded-exec/admission-queue.d.ts +68 -0
  10. package/dist/executors/bounded-exec/admission-queue.d.ts.map +1 -0
  11. package/dist/executors/bounded-exec/admission-queue.js +131 -0
  12. package/dist/executors/bounded-exec/cpu-grant.d.ts +14 -0
  13. package/dist/executors/bounded-exec/cpu-grant.d.ts.map +1 -0
  14. package/dist/executors/bounded-exec/cpu-grant.js +18 -0
  15. package/dist/executors/bounded-exec/cpu-pool.d.ts +1 -14
  16. package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -1
  17. package/dist/executors/bounded-exec/cpu-pool.js +2 -49
  18. package/dist/executors/bounded-exec/executor.d.ts +5 -12
  19. package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
  20. package/dist/executors/bounded-exec/executor.js +51 -154
  21. package/dist/executors/bounded-exec/memory-retry.d.ts +39 -0
  22. package/dist/executors/bounded-exec/memory-retry.d.ts.map +1 -0
  23. package/dist/executors/bounded-exec/memory-retry.js +33 -0
  24. package/dist/executors/bounded-exec/parent-grant.d.ts +43 -0
  25. package/dist/executors/bounded-exec/parent-grant.d.ts.map +1 -0
  26. package/dist/executors/bounded-exec/parent-grant.js +37 -0
  27. package/dist/executors/bounded-exec/peak-charge.d.ts +17 -0
  28. package/dist/executors/bounded-exec/peak-charge.d.ts.map +1 -0
  29. package/dist/executors/bounded-exec/peak-charge.js +20 -0
  30. package/dist/executors/bounded-exec/pool-environment.d.ts +25 -0
  31. package/dist/executors/bounded-exec/pool-environment.d.ts.map +1 -0
  32. package/dist/executors/bounded-exec/pool-environment.js +34 -0
  33. package/dist/executors/bounded-exec/run-cpu-pool.d.ts.map +1 -1
  34. package/dist/executors/bounded-exec/run-cpu-pool.js +1 -1
  35. package/dist/executors/bounded-exec/verdict.d.ts +3 -18
  36. package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
  37. package/dist/executors/bounded-exec/verdict.js +4 -25
  38. package/dist/index.d.ts +4 -1
  39. package/dist/index.d.ts.map +1 -1
  40. package/dist/index.js +97 -31
  41. package/dist/managed-files/ci-workflow.d.ts +29 -5
  42. package/dist/managed-files/ci-workflow.d.ts.map +1 -1
  43. package/dist/managed-files/ci-workflow.js +111 -16
  44. package/dist/managed-files/context.d.ts.map +1 -1
  45. package/dist/managed-files/context.js +1 -0
  46. package/dist/managed-files/files.d.ts +3 -1
  47. package/dist/managed-files/files.d.ts.map +1 -1
  48. package/dist/managed-files/files.js +2 -0
  49. package/dist/managed-files/publish-workflow.d.ts +6 -1
  50. package/dist/managed-files/publish-workflow.d.ts.map +1 -1
  51. package/dist/managed-files/publish-workflow.js +16 -11
  52. package/dist/owned-installation.d.ts.map +1 -1
  53. package/dist/owned-installation.js +48 -14
  54. package/dist/typecheck-route.d.ts +28 -0
  55. package/dist/typecheck-route.d.ts.map +1 -0
  56. package/dist/typecheck-route.js +109 -0
  57. package/dist/workspace-manifest.d.ts +50 -5
  58. package/dist/workspace-manifest.d.ts.map +1 -1
  59. package/executors.json +10 -0
  60. package/managed/raw/patches/ttsc@0.34.1.patch +93 -8
  61. package/managed/raw/tooling/direnv/apple-developer.sh +8 -1
  62. package/managed/raw/tooling/direnv/compiler-variables.sh +1 -1
  63. package/managed/raw/tooling/direnv/github-actions-bootstrap.sh +4 -2
  64. package/managed/raw/tooling/direnv/secret-references.ts +19 -5
  65. package/managed/raw/tooling/git-hooks/pre-push.sh +20 -10
  66. package/managed/templates/github/actions/forgejo-artifact.cjs +9 -9
  67. package/managed/templates/github/actions/setup-devenv/action.yml +53 -0
  68. package/package.json +2 -2
  69. package/src/built-output-packages.ts +125 -0
  70. package/src/cross-check-policy.ts +14 -1
  71. package/src/executors/bounded-exec/admission-queue.ts +163 -0
  72. package/src/executors/bounded-exec/cpu-grant.ts +32 -0
  73. package/src/executors/bounded-exec/cpu-pool.ts +2 -33
  74. package/src/executors/bounded-exec/executor.ts +60 -201
  75. package/src/executors/bounded-exec/memory-retry.ts +63 -0
  76. package/src/executors/bounded-exec/parent-grant.ts +75 -0
  77. package/src/executors/bounded-exec/peak-charge.ts +25 -0
  78. package/src/executors/bounded-exec/pool-environment.ts +45 -0
  79. package/src/executors/bounded-exec/run-cpu-pool.ts +2 -1
  80. package/src/executors/bounded-exec/verdict.ts +7 -35
  81. package/src/index.ts +105 -31
  82. package/src/managed-files/ci-workflow.ts +143 -16
  83. package/src/managed-files/context.ts +1 -0
  84. package/src/managed-files/files.ts +5 -0
  85. package/src/managed-files/publish-workflow.ts +26 -10
  86. package/src/owned-installation.ts +63 -14
  87. package/src/typecheck-route.ts +64 -0
  88. package/src/workspace-manifest.ts +56 -5
  89. package/test-timings.json +141 -139
@@ -13,31 +13,13 @@ import {
13
13
  processGroupHasMembers,
14
14
  signalProcessGroup,
15
15
  } from '../../process-group.js';
16
- import {
17
- admissionWaitMs,
18
- type BoundEvent,
19
- type BoundKind,
20
- type Bounds,
21
- earliestBound,
22
- NO_ADMISSION_WAIT,
23
- stepAdmissionWait,
24
- stepBounds,
25
- UNARMED,
26
- } from './bounds.js';
27
- import {
28
- CPU_POOL_ENV,
29
- type CpuPool,
30
- createCpuPool,
31
- type InheritedCpuPool,
32
- readInheritedCpuPool,
33
- type TokenGrant,
34
- takeParentCpuTokens,
35
- } from './cpu-pool.js';
16
+ import { AdmissionObservation, type QueueHost } from './admission-queue.js';
17
+ import { type BoundEvent, type BoundKind, type Bounds, earliestBound, stepBounds, UNARMED } from './bounds.js';
18
+ import { type CpuPool, createCpuPool } from './cpu-pool.js';
36
19
  import { GATEWAY_SOCKET } from './gateway-lease.js';
37
20
  import type { HangDiagnostics } from './hang-diagnostics.js';
38
21
  import { captureHangDiagnostics, HANG_CAPTURE_MS } from './hang-diagnostics.js';
39
22
  import { type LeafPeak, PEAK_FD, peakWaiter, readWaiterReport, underWaiter } from './leaf-peak.js';
40
- import type { CpuWaitRow } from './lease-wait.js';
41
23
  import type { LockWait } from './lock-wait.js';
42
24
  import {
43
25
  askedCores,
@@ -48,8 +30,8 @@ import {
48
30
  withCountedReservation,
49
31
  withMbxLeaf,
50
32
  } from './mbx-pool.js';
51
- import { MBX_QUEUE_POLL_MS, type MbxQueuedRequest, snapshotMbxQueue } from './mbx-queue.js';
52
33
  import { describeUnserved, RESERVATIONS_ENV, type Reservation, reserveCores, runEndpoint } from './mbx-reservations.js';
34
+ import { killedFromOutside, type RunOutcome, runUntilNotKilledForMemory } from './memory-retry.js';
53
35
  import { createNoSpaceWatch } from './no-space.js';
54
36
  import {
55
37
  COWSHED_EXECUTABLE_ENV,
@@ -60,6 +42,9 @@ import {
60
42
  type SubtreeAdmission,
61
43
  type SubtreeOwner,
62
44
  } from './owned-subtree.js';
45
+ import { askParentPool, describeGrant } from './parent-grant.js';
46
+ import { type LastPeak, readLastPeak } from './peak-charge.js';
47
+ import { CPU_POOL_ENV, type InheritedCpuPool, readInheritedCpuPool } from './pool-environment.js';
63
48
  import { createRunProgress, withProgressReport, withTestResultReport } from './progress.js';
64
49
  import {
65
50
  createHostCommands,
@@ -79,12 +64,10 @@ import { checkRuntimePin, describeRuntimePinRefusal } from './runtime-pin.js';
79
64
  import type { BoundedExecOptions } from './schema.js';
80
65
  import { createTestResultWriter, type TestResultWriter } from './test-result-writer.js';
81
66
  import {
82
- type AdmissionQueueWait,
83
67
  type BoundedExecRecord,
84
68
  type BoundedExecTask,
85
69
  BUN_TEST,
86
70
  clearTaskDirectory,
87
- describeAdmissionQueue,
88
71
  describeNoSpace,
89
72
  type ExpiredBound,
90
73
  JUNIT_FILE,
@@ -93,7 +76,6 @@ import {
93
76
  judgeRun,
94
77
  type NoSpaceSighting,
95
78
  readJunitReport,
96
- readLastPeak,
97
79
  taskDirectory,
98
80
  withJunitReport,
99
81
  writeRecord,
@@ -131,15 +113,7 @@ export interface NxTaskContext {
131
113
  taskGraph?: { tasks: Record<string, { hash?: string }> };
132
114
  }
133
115
 
134
- export interface BoundedExecResult {
135
- success: boolean;
136
- terminalOutput: string;
137
- /**
138
- * The command died of a SIGKILL that no bound of this executor and no signal it sent caused: the kernel's OOM killer,
139
- * which sends it to a leaf in a run's cgroup. Set on the last run of the command only when it was killed again.
140
- */
141
- killedForMemory?: boolean;
142
- }
116
+ export type BoundedExecResult = RunOutcome;
143
117
 
144
118
  interface RunState {
145
119
  settled: boolean;
@@ -157,8 +131,6 @@ interface RunState {
157
131
  lockWaits: readonly LockWait[];
158
132
  }
159
133
 
160
- type GrantedTokens = Extract<TokenGrant, { granted: true }>;
161
-
162
134
  export interface ProcessTreeKiller {
163
135
  kill(pid: number, signal: NodeJS.Signals): Promise<void>;
164
136
  }
@@ -318,39 +290,27 @@ export async function runBoundedExec(
318
290
  inherited: InheritedCpuPool | null = null,
319
291
  captureDiagnostics: typeof captureHangDiagnostics = captureHangDiagnostics,
320
292
  owner: SubtreeOwner | null = null,
321
- queueHost?: Parameters<typeof snapshotMbxQueue>[3],
293
+ queueHost?: QueueHost,
322
294
  ): Promise<BoundedExecResult> {
323
295
  const directory = context.record?.directory ?? null;
324
296
  // The task's last recorded peak, and the tokens it was lent when the peak was measured, are its charge in its parent's
325
297
  // pool, read once before a run's clear removes them: a command run again after a kill is charged by the record the
326
298
  // killed run replaced, not by the peak it reached before it was killed.
327
299
  const lastPeak = directory !== null && inherited !== null ? await readLastPeak(directory) : null;
328
- const run = (alone: boolean): Promise<BoundedExecResult> =>
329
- runCommand(options, context, killer, tempVolume, inherited, captureDiagnostics, owner, queueHost, {
330
- lastPeak,
331
- alone,
332
- });
333
- const first = await run(false);
334
- if (first.killedForMemory !== true || inherited === null) {
335
- return first;
336
- }
337
- // The kernel killed the leaf: a SIGKILL this executor did not send. It runs once more, alone in its run's pool, so
338
- // what it was killed beside is not there to be killed with it.
339
- const note = `cowshed: the kernel killed the command for memory (a SIGKILL this executor did not send${
340
- context.record === undefined ? '' : `: ${context.record.task.id}`
341
- }); running it once more, alone in its run's pool\n`;
342
- process.stderr.write(note);
343
- const again = await run(true);
344
- return {
345
- success: again.success,
346
- terminalOutput: `${first.terminalOutput}${note}${again.terminalOutput}`,
347
- ...(again.killedForMemory === true ? { killedForMemory: true } : {}),
348
- };
300
+ return runUntilNotKilledForMemory(
301
+ (alone) =>
302
+ runCommand(options, context, killer, tempVolume, inherited, captureDiagnostics, owner, queueHost, {
303
+ lastPeak,
304
+ alone,
305
+ }),
306
+ inherited !== null,
307
+ context.record?.task.id,
308
+ );
349
309
  }
350
310
 
351
311
  /** What a run of the command is told that the run before it was not: the task's charge, and whether it runs alone. */
352
312
  interface Attempt {
353
- readonly lastPeak: Awaited<ReturnType<typeof readLastPeak>> | null;
313
+ readonly lastPeak: LastPeak | null;
354
314
  readonly alone: boolean;
355
315
  }
356
316
 
@@ -363,7 +323,7 @@ async function runCommand(
363
323
  inherited: InheritedCpuPool | null,
364
324
  captureDiagnostics: typeof captureHangDiagnostics,
365
325
  owner: SubtreeOwner | null,
366
- queueHost: Parameters<typeof snapshotMbxQueue>[3] | undefined,
326
+ queueHost: QueueHost | undefined,
367
327
  attempt: Attempt,
368
328
  ): Promise<BoundedExecResult> {
369
329
  const cwd = resolveCwd(options.cwd, context.root);
@@ -608,24 +568,15 @@ async function runCommand(
608
568
  const asked = Date.now();
609
569
  // A pool that answered and refused is a live budget, and running beside it would count the run twice: the run fails
610
570
  // instead. A pool whose endpoint is gone (its holder exited) lends nothing, and the run goes on as asked.
611
- let grant: GrantedTokens | null = null;
612
- let parentRefusal: string | null = null;
613
- let parentWait: CpuWaitRow | undefined;
614
- if (inherited !== null) {
615
- const fromParent = await takeParentCpuTokens(inherited, demand.want, charge, {
616
- parallelism: chargeParallelism ?? undefined,
617
- alone: attempt.alone ? true : undefined,
618
- task: context.record?.task.id,
619
- });
620
- if (fromParent.granted) {
621
- grant = fromParent;
622
- } else if (fromParent.cause === 'absent') {
623
- appendStderr(`cowshed: cpu-pool parent pool absent: ${fromParent.reason}; running the command as asked\n`);
624
- } else {
625
- parentRefusal = fromParent.reason;
626
- parentWait = fromParent.wait;
627
- }
628
- }
571
+ const parent = await askParentPool(inherited, {
572
+ want: demand.want,
573
+ charge,
574
+ parallelism: chargeParallelism,
575
+ alone: attempt.alone,
576
+ task: context.record?.task.id,
577
+ });
578
+ for (const said of parent.said) appendStderr(said);
579
+ const { grant, refusal: parentRefusal, wait: parentWait } = parent;
629
580
  const waitedMs = Date.now() - asked;
630
581
  let cpu: BoundedExecRecord['cpu'];
631
582
  let sized: { command: string; env: Record<string, string> } = { command: requested, env: {} };
@@ -636,13 +587,8 @@ async function runCommand(
636
587
  if (grant !== null) {
637
588
  sized = sizeRunner(demand, requested, grant.tokens);
638
589
  cpu = { tokens: grant.tokens, want: demand.want, waitedMs };
639
- appendStderr(
640
- `cowshed: cpu-pool wait done elapsed=${waitedMs}ms status=ok tokens=${grant.tokens}/${demand.want} runner=${demand.kind} memory=${charge ?? 'none'}${grant.charged !== null && grant.charged !== charge ? ` charged=${grant.charged}` : ''}\n`,
641
- );
642
- if (grant.oversized !== null) {
643
- appendStderr(
644
- `cowshed: cpu-pool granted while nothing else holds memory: this task's last peak of ${charge} bytes alone exceeds the run's ${grant.oversized.memory} bytes\n`,
645
- );
590
+ for (const said of describeGrant(grant, { waitedMs, want: demand.want, runner: demand.kind, charge })) {
591
+ appendStderr(said);
646
592
  }
647
593
  } else if (leaf?.kind === 'reserve') {
648
594
  sized = { command: sizeRunnerToGrant(leaf.demand, requested), env: {} };
@@ -670,33 +616,15 @@ async function runCommand(
670
616
  })();
671
617
  return cpuReleased;
672
618
  };
673
- let mbxAdmitted = false;
674
- let admissionWait = NO_ADMISSION_WAIT;
675
- let lastQueueRequests: readonly MbxQueuedRequest[] = [];
676
- /**
677
- * Why the last admission snapshot failed, while the one after it has not succeeded; null otherwise. A snapshot only
678
- * decides whether waiting on mbx admission is credited against the idle bound, so a failed one withholds that credit
679
- * and ends nothing: the bounds decide, and an idle expiry names it.
680
- */
681
- let unconfirmedAdmission: string | null = null;
682
- /** When the last successful snapshot completed (`Date.now()`): credit runs to it and no further. */
683
- let confirmedAdmissionAt: number | null = null;
684
- // A reserved leaf's own request on its run's endpoint is its admission wait; a Cargo leaf's comes from mbx's report.
685
- const queueWait = (): AdmissionQueueWait | null =>
686
- mbxAdmitted || (lastQueueRequests.length === 0 && reservation === null)
687
- ? null
688
- : {
689
- waitedMs: admissionWaitMs(admissionWait, performance.now()),
690
- requests: lastQueueRequests,
691
- };
692
- const queueAbort = new AbortController();
693
- let queueTimer: NodeJS.Timeout | undefined;
694
- let queueSnapshot: Promise<void> | null = null;
695
- const stopQueueObservation = (): void => {
696
- admissionWait = stepAdmissionWait(admissionWait, { kind: 'stop', at: performance.now() });
697
- clearInterval(queueTimer);
698
- queueAbort.abort();
699
- };
619
+ const queue = new AdmissionObservation({
620
+ over: () => state.settled || state.expiry !== null,
621
+ pid: () => child.pid,
622
+ environment: () => mergeEnv(lease?.directory ?? ordinaryTemp, env),
623
+ tell,
624
+ step: (event) => stepTo(event),
625
+ testing: () => isTesting(),
626
+ queueHost,
627
+ });
700
628
  // Records the run's verdict when it has a task to record as. An `expiry` is set only when a bound ended the run.
701
629
  // Observation and ownership failures remain operational even after valid queue evidence; cancellation alone does not.
702
630
  const recordRun = async (
@@ -730,7 +658,7 @@ async function runCommand(
730
658
  appendStderr(`bounded-exec: ${error}\n`);
731
659
  }
732
660
  });
733
- const admissionQueue = queueWait();
661
+ const admissionQueue = queue.queueWait(reservation !== null);
734
662
  const verdict =
735
663
  operational || resultFailure !== null
736
664
  ? judgeOperationalFailure(exitCode, report)
@@ -1019,7 +947,7 @@ async function runCommand(
1019
947
  try {
1020
948
  // Drain an older poll, then take a fresh bounded snapshot before idle can end queued work. Total and fixture
1021
949
  // ceilings remain armed while this observation runs.
1022
- await refreshQueue(true);
950
+ await queue.refresh(true);
1023
951
  } finally {
1024
952
  checkingIdle = false;
1025
953
  }
@@ -1049,74 +977,6 @@ async function runCommand(
1049
977
  armBoundTimer();
1050
978
  };
1051
979
 
1052
- let lastQueueDiagnostic = '';
1053
- let lastQueueDiagnosticAt: number | null = null;
1054
- let lastSnapshotDiagnostics = '';
1055
- const unconfirmAdmission = (detail: string): void => {
1056
- if (state.settled || state.expiry !== null || queueAbort.signal.aborted) return;
1057
- if (unconfirmedAdmission !== detail) {
1058
- tell(
1059
- `bounded-exec: mbx admission snapshot failed: ${detail}; admission is unconfirmed, so this wait earns no idle credit until a snapshot succeeds\n`,
1060
- );
1061
- }
1062
- unconfirmedAdmission = detail;
1063
- lastQueueRequests = [];
1064
- admissionWait = stepAdmissionWait(admissionWait, { kind: 'snapshot', at: performance.now(), waiting: false });
1065
- // The wait since the last confirmed snapshot is unconfirmed: the idle bound resumes from that snapshot.
1066
- stepTo({ kind: 'admission-queue', at: confirmedAdmissionAt ?? Date.now(), waiting: false, testing: isTesting() });
1067
- };
1068
- const refreshQueue = async (fresh: boolean): Promise<void> => {
1069
- if (fresh && queueSnapshot !== null) await queueSnapshot;
1070
- if (state.settled || state.expiry !== null || queueAbort.signal.aborted || child.pid === undefined) return;
1071
- const pid = child.pid;
1072
- queueSnapshot ??= (async () => {
1073
- const snapshot = await snapshotMbxQueue(
1074
- pid,
1075
- mergeEnv(lease?.directory ?? ordinaryTemp, env),
1076
- queueAbort.signal,
1077
- queueHost,
1078
- );
1079
- if (state.settled || state.expiry !== null || queueAbort.signal.aborted) return;
1080
- if (!snapshot.ok) {
1081
- unconfirmAdmission(snapshot.detail);
1082
- return;
1083
- }
1084
- const diagnostics = snapshot.diagnostics?.join('; ') ?? '';
1085
- if (diagnostics !== '' && diagnostics !== lastSnapshotDiagnostics) tell(`bounded-exec: ${diagnostics}\n`);
1086
- lastSnapshotDiagnostics = diagnostics;
1087
- const waiting = snapshot.requests.length > 0;
1088
- const observedAt = Date.now();
1089
- const observedWaitAt = performance.now();
1090
- admissionWait = stepAdmissionWait(admissionWait, { kind: 'snapshot', at: observedWaitAt, waiting });
1091
- unconfirmedAdmission = null;
1092
- confirmedAdmissionAt = observedAt;
1093
- if (waiting) {
1094
- lastQueueRequests = snapshot.requests;
1095
- const positions = JSON.stringify(snapshot.requests);
1096
- if (
1097
- positions !== lastQueueDiagnostic ||
1098
- lastQueueDiagnosticAt === null ||
1099
- Date.now() - lastQueueDiagnosticAt >= HANG_CAPTURE_MS
1100
- ) {
1101
- const waitedMs = admissionWaitMs(admissionWait, observedWaitAt);
1102
- tell(`cowshed: mbx ${describeAdmissionQueue({ waitedMs, requests: snapshot.requests })}\n`);
1103
- lastQueueDiagnostic = positions;
1104
- lastQueueDiagnosticAt = Date.now();
1105
- }
1106
- } else {
1107
- lastQueueDiagnostic = '';
1108
- }
1109
- stepTo({ kind: 'admission-queue', at: observedAt, waiting, testing: isTesting() });
1110
- })()
1111
- .catch((error: unknown) => {
1112
- unconfirmAdmission(error instanceof Error ? error.message : String(error));
1113
- })
1114
- .finally(() => {
1115
- queueSnapshot = null;
1116
- });
1117
- await queueSnapshot;
1118
- };
1119
-
1120
980
  // Only the CHILD's own output is read for progress. The diagnostics below go
1121
981
  // through appendStderr as well, and re-arming on those would let a teardown
1122
982
  // message extend the very bound that just fired.
@@ -1208,9 +1068,9 @@ async function runCommand(
1208
1068
  clearTimeout(boundTimer);
1209
1069
  // Preserve explicit live membership before cancellation removes the queued process. A failed bounded observation
1210
1070
  // leaves the admission unconfirmed, never a manufactured queue or admission.
1211
- signalCleanup = (leaf?.kind !== 'cargo' ? Promise.resolve() : refreshQueue(false))
1071
+ signalCleanup = (leaf?.kind !== 'cargo' ? Promise.resolve() : queue.refresh(false))
1212
1072
  .finally(() => {
1213
- stopQueueObservation();
1073
+ queue.stop();
1214
1074
  return killChildTree();
1215
1075
  })
1216
1076
  .finally(releaseCpu)
@@ -1252,12 +1112,12 @@ async function runCommand(
1252
1112
  child.once('error', (error) => {
1253
1113
  appendStderr(`${error.message}\n`);
1254
1114
  state.settled = true;
1255
- stopQueueObservation();
1115
+ queue.stop();
1256
1116
  resolve({ code: 1, signal: null });
1257
1117
  });
1258
1118
  child.once(hooked === null ? 'exit' : 'close', (code: number | null, signal: NodeJS.Signals | null) => {
1259
1119
  state.settled = true;
1260
- stopQueueObservation();
1120
+ queue.stop();
1261
1121
  resolve({ code, signal });
1262
1122
  });
1263
1123
  });
@@ -1281,9 +1141,9 @@ async function runCommand(
1281
1141
  ? `still running: ${bound.tests.join(', ')}`
1282
1142
  : 'the runner named no test still running'
1283
1143
  } (cwd=${cwd}): ${command}\n${
1284
- unconfirmedAdmission === null
1144
+ queue.unconfirmed === null
1285
1145
  ? ''
1286
- : `The last mbx admission snapshot failed (${unconfirmedAdmission}), so the wait it could not confirm earned no idle credit\n`
1146
+ : `The last mbx admission snapshot failed (${queue.unconfirmed}), so the wait it could not confirm earned no idle credit\n`
1287
1147
  }`
1288
1148
  : bound.kind === 'fixture'
1289
1149
  ? `\nFixture command timed out after ${elapsedMs}ms (fixtureTimeoutMs=${bound.limitMs}, cwd=${cwd}): ${segment ?? command}\n`
@@ -1420,7 +1280,7 @@ async function runCommand(
1420
1280
  channel.end();
1421
1281
  return;
1422
1282
  }
1423
- admissionWait = stepAdmissionWait(admissionWait, { kind: 'snapshot', at: performance.now(), waiting: true });
1283
+ queue.waiting();
1424
1284
  tell(`cowshed: mbx reserve wait start cpus=${cpus} priority=${reservedLeaf.priority}\n`);
1425
1285
  const held = reserveCores(endpoint, cpus, reservedLeaf.priority);
1426
1286
  reservation = held;
@@ -1432,7 +1292,7 @@ async function runCommand(
1432
1292
  }
1433
1293
  // A run that ended while it waited gives the grant back with its CPU (`releaseCpu`).
1434
1294
  if (state.settled || state.expiry !== null) return;
1435
- mbxAdmitted = true;
1295
+ queue.admitted = true;
1436
1296
  tell(`cowshed: mbx reserve granted elapsed=${Date.now() - startedAt}ms priority=${reservedLeaf.priority}\n`);
1437
1297
  channel.end(`${granted.cpus}\n`);
1438
1298
  stepTo({ kind: 'granted', at: Date.now(), testing: isTesting(), fixture: !fixtureDone });
@@ -1451,7 +1311,7 @@ async function runCommand(
1451
1311
  for (const byte of chunk) {
1452
1312
  if (byte === 103) {
1453
1313
  if (!said) {
1454
- mbxAdmitted = true;
1314
+ queue.admitted = true;
1455
1315
  said = true;
1456
1316
  tell(`cowshed: mbx cargo granted elapsed=${Date.now() - startedAt}ms\n`);
1457
1317
  }
@@ -1465,10 +1325,7 @@ async function runCommand(
1465
1325
  }
1466
1326
  });
1467
1327
  // Only mbx's activity report says whether a Cargo leaf's compiler or test waits in the pool.
1468
- queueTimer = setInterval(() => {
1469
- void refreshQueue(false);
1470
- }, MBX_QUEUE_POLL_MS);
1471
- void refreshQueue(false);
1328
+ queue.poll();
1472
1329
  }
1473
1330
  }
1474
1331
 
@@ -1493,8 +1350,8 @@ async function runCommand(
1493
1350
  await signalCleanup;
1494
1351
  return { success: false, terminalOutput: outputChunks.join('') };
1495
1352
  }
1496
- stopQueueObservation();
1497
- await queueSnapshot;
1353
+ queue.stop();
1354
+ await queue.idle();
1498
1355
  await termination;
1499
1356
  clearTimeout(boundTimer);
1500
1357
  cancelCapture();
@@ -1549,10 +1406,12 @@ async function runCommand(
1549
1406
  // branch later exited 0: its verdict is an ordinary operational failure, never a successful or silently unbounded run.
1550
1407
  const operationalFailure = state.poolCloseError !== null || lifecycleFailure !== null || state.retireError !== null;
1551
1408
  await recordRun(code, state.expiry, Date.now() - startedAt, operationalFailure, seen, state.lockWaits, peakBytes);
1552
- // A SIGKILL this executor did not send, to a command that no bound of this executor ended, came from outside it. In a
1553
- // run's cgroup the one that sends it to a leaf is the kernel's OOM killer. The exit code alone says nothing of who
1554
- // sent it: a bound's own force-kill ends in 137 too, and is why `sentSignal` is read first.
1555
- const killedForMemory = !state.sentSignal && state.expiry === null && (exitSignal === 'SIGKILL' || code === 137);
1409
+ const killedForMemory = killedFromOutside({
1410
+ sentSignal: state.sentSignal,
1411
+ bounded: state.expiry !== null,
1412
+ exitSignal,
1413
+ code,
1414
+ });
1556
1415
  return {
1557
1416
  success: !operationalFailure && resultFailure === null && state.expiry === null && code === 0,
1558
1417
  terminalOutput: outputChunks.join(''),
@@ -0,0 +1,63 @@
1
+ /**
2
+ * A run again after the kernel killed the command for memory (specs/cowshed/05_gateway.md, "Host scheduling"). Only
3
+ * when a pool lent the run its tokens and a SIGKILL that no bound of this executor and no signal it sent caused ended
4
+ * it: the command runs once more, alone in its run's pool, so what it was killed beside is not there to be killed with
5
+ * it. This decides whether a command runs a second time and how its two outputs join, never what it runs: the key
6
+ * leaves it out (README, "Bounded exec key").
7
+ */
8
+
9
+ /** How one run of the command ended. */
10
+ export interface RunOutcome {
11
+ success: boolean;
12
+ terminalOutput: string;
13
+ /**
14
+ * The command died of a SIGKILL that no bound of this executor and no signal it sent caused: the kernel's OOM killer,
15
+ * which sends it to a leaf in a run's cgroup. Set on the last run of the command only when it was killed again.
16
+ */
17
+ killedForMemory?: boolean;
18
+ }
19
+
20
+ /** What a run read of how its command ended and what this executor did to it. */
21
+ export interface Ending {
22
+ /** This executor sent the command's process group a signal. */
23
+ readonly sentSignal: boolean;
24
+ /** A bound of this executor ended the run. */
25
+ readonly bounded: boolean;
26
+ readonly exitSignal: NodeJS.Signals | null;
27
+ /** The exit status, or the status a signal folds to. */
28
+ readonly code: number;
29
+ }
30
+
31
+ /**
32
+ * A SIGKILL this executor did not send, to a command that no bound of this executor ended, came from outside it. In a
33
+ * run's cgroup the one that sends it to a leaf is the kernel's OOM killer. The exit code alone says nothing of who sent
34
+ * it: a bound's own force-kill ends in 137 too, and is why `sentSignal` is read first.
35
+ */
36
+ export function killedFromOutside(ending: Ending): boolean {
37
+ return !ending.sentSignal && !ending.bounded && (ending.exitSignal === 'SIGKILL' || ending.code === 137);
38
+ }
39
+
40
+ /**
41
+ * Run the command; when the kernel killed it for memory and a pool lent it tokens, run it once more alone, and join the
42
+ * two runs' output with a note naming the task.
43
+ */
44
+ export async function runUntilNotKilledForMemory(
45
+ run: (alone: boolean) => Promise<RunOutcome>,
46
+ pooled: boolean,
47
+ task: string | undefined,
48
+ ): Promise<RunOutcome> {
49
+ const first = await run(false);
50
+ if (first.killedForMemory !== true || !pooled) {
51
+ return first;
52
+ }
53
+ const note = `cowshed: the kernel killed the command for memory (a SIGKILL this executor did not send${
54
+ task === undefined ? '' : `: ${task}`
55
+ }); running it once more, alone in its run's pool\n`;
56
+ process.stderr.write(note);
57
+ const again = await run(true);
58
+ return {
59
+ success: again.success,
60
+ terminalOutput: `${first.terminalOutput}${note}${again.terminalOutput}`,
61
+ ...(again.killedForMemory === true ? { killedForMemory: true } : {}),
62
+ };
63
+ }
@@ -0,0 +1,75 @@
1
+ import { type TokenGrant, takeParentCpuTokens } from './cpu-pool.js';
2
+ import type { CpuWaitRow } from './lease-wait.js';
3
+ import type { InheritedCpuPool } from './pool-environment.js';
4
+
5
+ /**
6
+ * What a run asks its parent's CPU pool and says of the answer. How many tokens the pool lends, how long the run waits
7
+ * and what it is charged are values that differ between runs of one task key; what a token count does to the command is
8
+ * `runner-sizing.ts`'s, keyed (README, "Bounded exec key").
9
+ */
10
+
11
+ /** Tokens a parent pool lent, held until released. */
12
+ export type GrantedTokens = Extract<TokenGrant, { granted: true }>;
13
+
14
+ /** The parent pool's answer to a run: a grant, a refusal that fails the run, or neither, and what to say of it. */
15
+ export interface ParentGrant {
16
+ readonly grant: GrantedTokens | null;
17
+ /** A pool that answered and refused is a live budget, and the run fails rather than run beside it. */
18
+ readonly refusal: string | null;
19
+ readonly wait: CpuWaitRow | undefined;
20
+ /** What the run says, in order. */
21
+ readonly said: readonly string[];
22
+ }
23
+
24
+ /** What a run asks for: its count, what its task's last recorded peak charges it, and whether it runs alone. */
25
+ export interface Ask {
26
+ readonly want: number;
27
+ readonly charge: number | null;
28
+ /** The tokens the task was lent when its recorded peak was measured. */
29
+ readonly parallelism: number | null;
30
+ readonly alone: boolean;
31
+ readonly task: string | undefined;
32
+ }
33
+
34
+ /**
35
+ * Ask the parent pool for the run's tokens. A run that inherits no pool runs as asked and says nothing; a pool whose
36
+ * endpoint is gone (its holder exited) lends nothing, and the run goes on as asked.
37
+ */
38
+ export async function askParentPool(inherited: InheritedCpuPool | null, ask: Ask): Promise<ParentGrant> {
39
+ if (inherited === null) {
40
+ return { grant: null, refusal: null, wait: undefined, said: [] };
41
+ }
42
+ const answer = await takeParentCpuTokens(inherited, ask.want, ask.charge, {
43
+ parallelism: ask.parallelism ?? undefined,
44
+ alone: ask.alone ? true : undefined,
45
+ task: ask.task,
46
+ });
47
+ if (answer.granted) {
48
+ return { grant: answer, refusal: null, wait: undefined, said: [] };
49
+ }
50
+ if (answer.cause === 'absent') {
51
+ return {
52
+ grant: null,
53
+ refusal: null,
54
+ wait: undefined,
55
+ said: [`cowshed: cpu-pool parent pool absent: ${answer.reason}; running the command as asked\n`],
56
+ };
57
+ }
58
+ return { grant: null, refusal: answer.reason, wait: answer.wait, said: [] };
59
+ }
60
+
61
+ /** What a run says once its grant is in hand. */
62
+ export function describeGrant(
63
+ grant: GrantedTokens,
64
+ run: { waitedMs: number; want: number; runner: string; charge: number | null },
65
+ ): string[] {
66
+ const said = [
67
+ `cowshed: cpu-pool wait done elapsed=${run.waitedMs}ms status=ok tokens=${grant.tokens}/${run.want} runner=${run.runner} memory=${run.charge ?? 'none'}${grant.charged !== null && grant.charged !== run.charge ? ` charged=${grant.charged}` : ''}\n`,
68
+ ];
69
+ if (grant.oversized !== null) {
70
+ said.push(
71
+ `cowshed: cpu-pool granted while nothing else holds memory: this task's last peak of ${run.charge} bytes alone exceeds the run's ${grant.oversized.memory} bytes\n`,
72
+ );
73
+ }
74
+ return said;
75
+ }
@@ -0,0 +1,25 @@
1
+ import { join } from 'node:path';
2
+ import { readRecord, VERDICT_FILE } from './verdict.js';
3
+
4
+ /** What the task's last record charges its next run: the peak and the parallelism it was measured under. */
5
+ export type LastPeak =
6
+ | { ok: true; peakBytes: number | null; parallelism: number | null }
7
+ | { ok: false; error: string };
8
+
9
+ /**
10
+ * The peak the task's last record in `directory` names: its charge in its run's pool. Null when the task has no record
11
+ * or its record no peak; a record that cannot be read is an error naming it, never a charge of nothing. It decides what a
12
+ * pool charges a run, a value that differs between runs of one task key, so the key leaves it out (README, "Bounded
13
+ * exec key").
14
+ */
15
+ export async function readLastPeak(directory: string): Promise<LastPeak> {
16
+ try {
17
+ const record = await readRecord(directory);
18
+ return { ok: true, peakBytes: record?.peakBytes ?? null, parallelism: record?.cpu?.tokens ?? null };
19
+ } catch (error) {
20
+ return {
21
+ ok: false,
22
+ error: `the task's last record ${join(directory, VERDICT_FILE)} cannot be read: ${error instanceof Error ? error.message : String(error)}`,
23
+ };
24
+ }
25
+ }
@@ -0,0 +1,45 @@
1
+ import { CPU_TOKENS_ENV, parseInheritedTokens } from './runner-sizing.js';
2
+
3
+ /**
4
+ * What a CPU pool puts in a command's environment (cpu-pool.ts serves the pool; this module decides what the command
5
+ * sees of it, so it is keyed: executors.json lists cpu-pool.js as output-neutral, which it may only be while it writes
6
+ * no variable the command reads).
7
+ */
8
+
9
+ /** The environment name a child reads its parent's pool endpoint from; its count is `CPU_TOKENS_ENV`. */
10
+ export const CPU_POOL_ENV = 'BOUNDED_EXEC_CPU_POOL';
11
+
12
+ /** An inherited pool descriptor; only a grant from its endpoint proves a live allocation. */
13
+ export interface InheritedCpuPool {
14
+ readonly socket: string;
15
+ /** The count the parent names beside its endpoint. */
16
+ readonly tokens: number;
17
+ }
18
+
19
+ /**
20
+ * The inherited descriptor, or null when malformed or absent. A missing or bare count, or an empty
21
+ * or NUL-bearing endpoint, proves no allocation; the count is a positive whole number. A valid
22
+ * descriptor still needs a live endpoint grant.
23
+ */
24
+ export function readInheritedCpuPool(environment: Readonly<NodeJS.ProcessEnv>): InheritedCpuPool | null {
25
+ const socket = environment[CPU_POOL_ENV];
26
+ const tokens = parseInheritedTokens(environment[CPU_TOKENS_ENV]);
27
+ if (socket === undefined || socket.length === 0 || socket.includes('\0') || tokens === null) {
28
+ return null;
29
+ }
30
+ return { socket, tokens };
31
+ }
32
+
33
+ /**
34
+ * The environment a child of a pool inherits to reach it: its endpoint, the count the pool holds, and its make
35
+ * jobserver, which Cargo and make read from the fifo; `-j` with a jobserver names no count of its own.
36
+ */
37
+ export function poolEnvironment(socketPath: string, tokens: number, fifoPath: string): Record<string, string> {
38
+ const jobserverFlags = `-j --jobserver-auth=fifo:${fifoPath}`;
39
+ return {
40
+ [CPU_POOL_ENV]: socketPath,
41
+ [CPU_TOKENS_ENV]: String(tokens),
42
+ CARGO_MAKEFLAGS: jobserverFlags,
43
+ MAKEFLAGS: jobserverFlags,
44
+ };
45
+ }
@@ -3,10 +3,11 @@ import { fileURLToPath } from 'node:url';
3
3
  import { Worker } from 'node:worker_threads';
4
4
  import { logger } from 'nx/src/devkit-exports.js';
5
5
  import type { PostTasksExecutionContext, PreTasksExecutionContext } from 'nx/src/project-graph/plugins/public-api.js';
6
- import { type CloseOutcome, type CpuPool, readInheritedCpuPool } from './cpu-pool.js';
6
+ import type { CloseOutcome, CpuPool } from './cpu-pool.js';
7
7
  import type { CpuPoolThreadData, CpuPoolThreadReport } from './cpu-pool-thread.js';
8
8
  import { type RunMemory, type RunMemoryReading, RunMemoryUnreadable, readRunMemory } from './host-memory.js';
9
9
  import { mbxPool } from './mbx-pool.js';
10
+ import { readInheritedCpuPool } from './pool-environment.js';
10
11
  import { CPU_TOKENS_ENV, parseInheritedTokens } from './runner-sizing.js';
11
12
 
12
13
  /**