@smoothbricks/nx-plugin 0.7.18 → 0.7.19

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (143) hide show
  1. package/README.md +160 -99
  2. package/dist/.build-inputs.json +35 -29
  3. package/dist/built-output-packages.js +1 -1
  4. package/dist/cargo-closure-input.js +1 -1
  5. package/dist/cargo-source-hash.d.ts +3 -21
  6. package/dist/cargo-source-hash.d.ts.map +1 -1
  7. package/dist/cargo-source-hash.js +4 -91
  8. package/dist/cargo-toolchain-paths.d.ts +2 -2
  9. package/dist/cargo-toolchain-paths.js +2 -2
  10. package/dist/cargo-workspace.d.ts.map +1 -1
  11. package/dist/cargo-workspace.js +2 -1
  12. package/dist/executors/bounded-exec/admission-queue.d.ts +3 -1
  13. package/dist/executors/bounded-exec/admission-queue.d.ts.map +1 -1
  14. package/dist/executors/bounded-exec/admission-queue.js +9 -10
  15. package/dist/executors/bounded-exec/awake-clock.d.ts +5 -0
  16. package/dist/executors/bounded-exec/awake-clock.d.ts.map +1 -0
  17. package/dist/executors/bounded-exec/awake-clock.js +22 -0
  18. package/dist/executors/bounded-exec/bounds.d.ts +7 -5
  19. package/dist/executors/bounded-exec/bounds.d.ts.map +1 -1
  20. package/dist/executors/bounded-exec/bounds.js +14 -11
  21. package/dist/executors/bounded-exec/case-gate.d.ts +114 -0
  22. package/dist/executors/bounded-exec/case-gate.d.ts.map +1 -0
  23. package/dist/executors/bounded-exec/case-gate.js +342 -0
  24. package/dist/executors/bounded-exec/cpu-pool-thread.d.ts +1 -0
  25. package/dist/executors/bounded-exec/cpu-pool-thread.d.ts.map +1 -1
  26. package/dist/executors/bounded-exec/cpu-pool-thread.js +2 -2
  27. package/dist/executors/bounded-exec/cpu-pool.d.ts +28 -1
  28. package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -1
  29. package/dist/executors/bounded-exec/cpu-pool.js +102 -16
  30. package/dist/executors/bounded-exec/endpoint-life.d.ts +146 -0
  31. package/dist/executors/bounded-exec/endpoint-life.d.ts.map +1 -0
  32. package/dist/executors/bounded-exec/endpoint-life.js +228 -0
  33. package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
  34. package/dist/executors/bounded-exec/executor.js +207 -69
  35. package/dist/executors/bounded-exec/gateway-lease.d.ts +1 -1
  36. package/dist/executors/bounded-exec/gateway-lease.d.ts.map +1 -1
  37. package/dist/executors/bounded-exec/gateway-lease.js +8 -1
  38. package/dist/executors/bounded-exec/go-run-summary.d.ts +51 -0
  39. package/dist/executors/bounded-exec/go-run-summary.d.ts.map +1 -0
  40. package/dist/executors/bounded-exec/go-run-summary.js +98 -0
  41. package/dist/executors/bounded-exec/hang-diagnostics.d.ts +3 -3
  42. package/dist/executors/bounded-exec/hang-diagnostics.d.ts.map +1 -1
  43. package/dist/executors/bounded-exec/hang-diagnostics.js +66 -48
  44. package/dist/executors/bounded-exec/lease-wait.d.ts +1 -0
  45. package/dist/executors/bounded-exec/lease-wait.d.ts.map +1 -1
  46. package/dist/executors/bounded-exec/lease-wait.js +1 -0
  47. package/dist/executors/bounded-exec/mbx-pool.d.ts +28 -7
  48. package/dist/executors/bounded-exec/mbx-pool.d.ts.map +1 -1
  49. package/dist/executors/bounded-exec/mbx-pool.js +30 -9
  50. package/dist/executors/bounded-exec/mbx-reservations.d.ts +8 -5
  51. package/dist/executors/bounded-exec/mbx-reservations.d.ts.map +1 -1
  52. package/dist/executors/bounded-exec/mbx-reservations.js +187 -283
  53. package/dist/executors/bounded-exec/parent-grant.d.ts +2 -0
  54. package/dist/executors/bounded-exec/parent-grant.d.ts.map +1 -1
  55. package/dist/executors/bounded-exec/parent-grant.js +1 -0
  56. package/dist/executors/bounded-exec/pool-admission.d.ts +29 -3
  57. package/dist/executors/bounded-exec/pool-admission.d.ts.map +1 -1
  58. package/dist/executors/bounded-exec/pool-admission.js +47 -1
  59. package/dist/executors/bounded-exec/progress.d.ts +3 -0
  60. package/dist/executors/bounded-exec/progress.d.ts.map +1 -1
  61. package/dist/executors/bounded-exec/progress.js +31 -1
  62. package/dist/executors/bounded-exec/reservation-connection.d.ts +90 -0
  63. package/dist/executors/bounded-exec/reservation-connection.d.ts.map +1 -0
  64. package/dist/executors/bounded-exec/reservation-connection.js +248 -0
  65. package/dist/executors/bounded-exec/run-cpu-pool.d.ts.map +1 -1
  66. package/dist/executors/bounded-exec/run-cpu-pool.js +3 -0
  67. package/dist/executors/bounded-exec/runner-sizing.d.ts +9 -5
  68. package/dist/executors/bounded-exec/runner-sizing.d.ts.map +1 -1
  69. package/dist/executors/bounded-exec/runner-sizing.js +13 -9
  70. package/dist/executors/bounded-exec/schema.d.ts +2 -2
  71. package/dist/executors/bounded-exec/suite-minimum.d.ts +21 -0
  72. package/dist/executors/bounded-exec/suite-minimum.d.ts.map +1 -0
  73. package/dist/executors/bounded-exec/suite-minimum.js +85 -0
  74. package/dist/executors/bounded-exec/test-result-writer.d.ts +2 -0
  75. package/dist/executors/bounded-exec/test-result-writer.d.ts.map +1 -1
  76. package/dist/executors/bounded-exec/test-result-writer.js +1 -0
  77. package/dist/executors/bounded-exec/test-results.d.ts +5 -0
  78. package/dist/executors/bounded-exec/test-results.d.ts.map +1 -1
  79. package/dist/executors/bounded-exec/test-results.js +2 -2
  80. package/dist/executors/bounded-exec/verdict.d.ts +26 -4
  81. package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
  82. package/dist/executors/bounded-exec/verdict.js +26 -5
  83. package/dist/executors/cargo-zigbuild/executor.d.ts +10 -8
  84. package/dist/executors/cargo-zigbuild/executor.d.ts.map +1 -1
  85. package/dist/executors/cargo-zigbuild/executor.js +32 -19
  86. package/dist/index.d.ts.map +1 -1
  87. package/dist/index.js +105 -73
  88. package/dist/managed-files/ci-workflow.d.ts +4 -1
  89. package/dist/managed-files/ci-workflow.d.ts.map +1 -1
  90. package/dist/managed-files/ci-workflow.js +9 -14
  91. package/dist/managed-files/files.d.ts +3 -3
  92. package/dist/managed-files/files.js +3 -3
  93. package/dist/task-results.js +2 -1
  94. package/dist/workspace-config-policy.d.ts.map +1 -1
  95. package/dist/workspace-config-policy.js +135 -42
  96. package/executors.json +2 -0
  97. package/managed/raw/envrc +24 -0
  98. package/managed/raw/patches/nx@24.0.3.patch +1179 -133
  99. package/managed/raw/patches/ttsc@0.37.1.patch +1055 -0
  100. package/managed/raw/tooling/direnv/developer-links.ts +105 -11
  101. package/managed/raw/tooling/direnv/devenv.smoo.nix +20 -19
  102. package/package.json +67 -11
  103. package/runtime/nx-publish/.build-inputs.json +2 -2
  104. package/runtime/nx-publish/cache_adapter.js +293 -3
  105. package/runtime/nx-publish/nx_publish.js +36 -14
  106. package/src/built-output-packages.ts +1 -1
  107. package/src/cargo-closure-input.ts +1 -1
  108. package/src/cargo-source-hash.ts +16 -132
  109. package/src/cargo-toolchain-paths.ts +2 -2
  110. package/src/cargo-workspace.ts +1 -2
  111. package/src/executors/bounded-exec/admission-queue.ts +11 -10
  112. package/src/executors/bounded-exec/awake-clock.ts +20 -0
  113. package/src/executors/bounded-exec/bounds.ts +28 -15
  114. package/src/executors/bounded-exec/case-gate.ts +365 -0
  115. package/src/executors/bounded-exec/cpu-pool-thread.ts +3 -3
  116. package/src/executors/bounded-exec/cpu-pool.ts +139 -12
  117. package/src/executors/bounded-exec/endpoint-life.ts +279 -0
  118. package/src/executors/bounded-exec/executor.ts +280 -106
  119. package/src/executors/bounded-exec/gateway-lease.ts +6 -0
  120. package/src/executors/bounded-exec/go-run-summary.ts +137 -0
  121. package/src/executors/bounded-exec/hang-diagnostics.ts +70 -52
  122. package/src/executors/bounded-exec/lease-wait.ts +1 -0
  123. package/src/executors/bounded-exec/mbx-pool.ts +36 -9
  124. package/src/executors/bounded-exec/mbx-reservations.ts +213 -182
  125. package/src/executors/bounded-exec/parent-grant.ts +3 -0
  126. package/src/executors/bounded-exec/pool-admission.ts +59 -2
  127. package/src/executors/bounded-exec/progress.ts +37 -2
  128. package/src/executors/bounded-exec/reservation-connection.ts +221 -0
  129. package/src/executors/bounded-exec/run-cpu-pool.ts +2 -0
  130. package/src/executors/bounded-exec/runner-sizing.ts +13 -8
  131. package/src/executors/bounded-exec/schema.json +1 -1
  132. package/src/executors/bounded-exec/schema.ts +2 -2
  133. package/src/executors/bounded-exec/suite-minimum.ts +81 -0
  134. package/src/executors/bounded-exec/test-result-writer.ts +3 -0
  135. package/src/executors/bounded-exec/test-results.ts +5 -0
  136. package/src/executors/bounded-exec/verdict.ts +39 -6
  137. package/src/executors/cargo-zigbuild/executor.ts +30 -19
  138. package/src/index.ts +112 -78
  139. package/src/managed-files/ci-workflow.ts +13 -17
  140. package/src/managed-files/files.ts +3 -3
  141. package/src/workspace-config-policy.ts +161 -43
  142. package/test-timings.json +157 -147
  143. package/managed/raw/patches/ttsc@0.36.3.patch +0 -1660
@@ -61,6 +61,7 @@ export function holdLease(
61
61
  what: string,
62
62
  bounds: LeaseBounds | CpuLeaseWait,
63
63
  onQueued?: () => void,
64
+ withdrawn?: AbortSignal,
64
65
  ): Promise<HeldLease> {
65
66
  // `Promise.withResolvers` would read better but needs lib es2024; this package inherits lib
66
67
  // es2022 from tsconfig.base.json.
@@ -231,5 +232,10 @@ export function holdLease(
231
232
  socketClosed = true;
232
233
  unleased(`the lease connection for ${what} at ${socket} closed`);
233
234
  });
235
+ if (withdrawn !== undefined) {
236
+ const withdraw = (): void => unleased(`the request for ${what} at ${socket} was withdrawn`);
237
+ if (withdrawn.aborted) withdraw();
238
+ else withdrawn.addEventListener('abort', withdraw, { once: true });
239
+ }
234
240
  return promise;
235
241
  }
@@ -0,0 +1,137 @@
1
+ import { spawnSync } from 'node:child_process';
2
+ import { readFileSync } from 'node:fs';
3
+ import { join, relative, sep } from 'node:path';
4
+
5
+ /**
6
+ * A Go run by its top-level tests. `go test -json` reports every event and sums none, and a consumer that keeps only
7
+ * part of a long output cannot count it, so the run that read every event states the sum where its output ends.
8
+ */
9
+ export interface GoRunSummary {
10
+ readonly passed: number;
11
+ readonly failed: number;
12
+ readonly skipped: number;
13
+ readonly packages: number;
14
+ readonly failedTests: readonly GoFailedTest[];
15
+ /** Packages that failed with no failed test of their own: a build that did not compile, a `TestMain` that exited. */
16
+ readonly failedPackages: readonly string[];
17
+ }
18
+
19
+ export interface GoFailedTest {
20
+ readonly package: string;
21
+ readonly test: string;
22
+ }
23
+
24
+ /** One package as `go list -f GO_LIST_FORMAT` prints it: its directory and its in-package and external test files. */
25
+ export interface GoPackageFiles {
26
+ readonly package: string;
27
+ readonly directory: string;
28
+ readonly files: readonly string[];
29
+ }
30
+
31
+ export const GO_LIST_FORMAT = '{{.ImportPath}}\t{{.Dir}}\t{{join .TestGoFiles ","}}\t{{join .XTestGoFiles ","}}';
32
+
33
+ export function parseGoList(stdout: string): GoPackageFiles[] {
34
+ const packages: GoPackageFiles[] = [];
35
+ for (const line of stdout.split('\n')) {
36
+ const [name, directory, own, external] = line.split('\t');
37
+ if (name === undefined || name === '' || directory === undefined || directory === '') continue;
38
+ packages.push({
39
+ package: name,
40
+ directory,
41
+ files: [own ?? '', external ?? ''].flatMap((list) => (list === '' ? [] : list.split(','))),
42
+ });
43
+ }
44
+ return packages;
45
+ }
46
+
47
+ /**
48
+ * The file of a package that declares the top-level function `test`, and its line. Go admits one such function per
49
+ * test package, and a directory holds two test packages (in-package and external): a name both declare has no one
50
+ * file, and neither does a name none declares.
51
+ */
52
+ export function declarationOf(
53
+ test: string,
54
+ files: readonly { readonly path: string; readonly text: string }[],
55
+ ): { path: string; line: number } | null {
56
+ const declares = new RegExp(`^func\\s+${test.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\s*\\(`);
57
+ let found: { path: string; line: number } | null = null;
58
+ for (const file of files) {
59
+ const line = file.text.split('\n').findIndex((text) => declares.test(text));
60
+ if (line === -1) continue;
61
+ if (found !== null) return null;
62
+ found = { path: file.path, line: line + 1 };
63
+ }
64
+ return found;
65
+ }
66
+
67
+ /** Each failed test whose declaration is found, as `<workspace path>:<line>`, keyed `<package>::<test>`. */
68
+ export function goDeclarations(
69
+ failed: readonly GoFailedTest[],
70
+ root: string,
71
+ listed: readonly GoPackageFiles[],
72
+ read: (path: string) => string | null,
73
+ ): Map<string, string> {
74
+ const declared = new Map<string, string>();
75
+ for (const { package: name, test } of failed) {
76
+ const found = listed.find((entry) => entry.package === name);
77
+ if (found === undefined) continue;
78
+ const files = found.files.flatMap((file) => {
79
+ const path = join(found.directory, file);
80
+ const text = read(path);
81
+ return text === null ? [] : [{ path, text }];
82
+ });
83
+ const declaration = declarationOf(test, files);
84
+ if (declaration === null) continue;
85
+ declared.set(`${name}::${test}`, `${relative(root, declaration.path).split(sep).join('/')}:${declaration.line}`);
86
+ }
87
+ return declared;
88
+ }
89
+
90
+ /**
91
+ * The lines a Go run's output ends with. A failed test is named at its declaration, the identity a reader matches a
92
+ * source file by; one whose declaration was not found keeps its package, which no reader takes for a file.
93
+ */
94
+ export function goRunSummaryText(summary: GoRunSummary, declared: ReadonlyMap<string, string>): string {
95
+ const lines = [
96
+ `go test summary: ${summary.passed} passed, ${summary.failed} failed, ${summary.skipped} skipped, ${summary.packages} packages`,
97
+ ...summary.failedTests.map(
98
+ ({ package: name, test }) => `go test failed: ${declared.get(`${name}::${test}`) ?? name} ${test}`,
99
+ ),
100
+ ...summary.failedPackages.map((name) => `go test package failed: ${name}`),
101
+ ];
102
+ return `${lines.join('\n')}\n`;
103
+ }
104
+
105
+ /** How long `go list` may take to name the failed packages' directories and test files. */
106
+ const GO_LIST_MS = 60_000;
107
+
108
+ /**
109
+ * The declarations of a run's failed tests, asked of the toolchain that ran them: `go list` names each failed package's
110
+ * directory and test files, and the file that declares the test is found among them. A list that does not answer
111
+ * leaves every test at its package, which the summary then prints.
112
+ */
113
+ export function declaredGoTests(
114
+ failed: readonly GoFailedTest[],
115
+ cwd: string,
116
+ root: string,
117
+ env: NodeJS.ProcessEnv,
118
+ ): Map<string, string> {
119
+ if (failed.length === 0) return new Map();
120
+ const packages = [...new Set(failed.map((test) => test.package))];
121
+ const listed = spawnSync('go', ['list', '-f', GO_LIST_FORMAT, ...packages], {
122
+ cwd,
123
+ env,
124
+ encoding: 'utf8',
125
+ timeout: GO_LIST_MS,
126
+ });
127
+ if (listed.status !== 0) return new Map();
128
+ return goDeclarations(failed, root, parseGoList(listed.stdout), (path) => {
129
+ try {
130
+ return readFileSync(path, 'utf8');
131
+ } catch (error) {
132
+ // A listed file that cannot be read declares nothing here; any other failure is a bug to see.
133
+ if (error instanceof Error && 'code' in error) return null;
134
+ throw error;
135
+ }
136
+ });
137
+ }
@@ -1,6 +1,7 @@
1
1
  import { spawn } from 'node:child_process';
2
2
  import { describeTeardown, killProcessGroupNow, TEARDOWN_PATIENCE } from '../../process-group.js';
3
3
  import { type ProcessEntry, withDescendants } from '../../testing.js';
4
+ import { atClockDeadline } from './awake-clock.js';
4
5
  import {
5
6
  CPU_RISING_CLASS,
6
7
  type CpuRise,
@@ -37,9 +38,9 @@ const PS_FIELDS = 'pid=,ppid=,pgid=,stat=,%cpu=,etime=,cputime=,lstart=,command=
37
38
  const CPU_INTERVAL_MS = 5_000;
38
39
  /**
39
40
  * Stack samples: `sample` suspends a process every millisecond for one second and reports its threads' call graph. The
40
- * runner and at most this many leaf processes are sampled, all at once. `-mayDie` is not passed: it reads symbols
41
- * before sampling and measured about a second slower under load, and a process that exits mid-sample is a raced
42
- * capture all the same.
41
+ * runner and at most this many leaf processes are sampled, all at once, and then at most this many named lock waiters
42
+ * those left out. `-mayDie` is not passed: it reads symbols before sampling and measured about a second slower under
43
+ * load, and a process that exits mid-sample is a raced capture all the same.
43
44
  */
44
45
  const MAX_SAMPLED_LEAVES = 4;
45
46
  const SAMPLE_SECONDS = 1;
@@ -87,8 +88,8 @@ export interface HangDiagnostics {
87
88
  export interface HangCaptureRequest {
88
89
  pid: number;
89
90
  pgid: number | null;
90
- expiresAtMs: number;
91
- deadlineMs: number;
91
+ awakeExpiresAtMs: number;
92
+ awakeNow(): number;
92
93
  signal?: AbortSignal;
93
94
  }
94
95
 
@@ -111,16 +112,14 @@ interface ObservedProcess extends ProcessEntry {
111
112
  raw: string;
112
113
  }
113
114
 
114
- const host: HangDiagnosticHost = { platform: process.platform, probe: runProbe, sleep: sleepUnlessAborted };
115
-
116
- function sleepUnlessAborted(ms: number, signal: AbortSignal): Promise<void> {
115
+ function sleepUnlessAborted(ms: number, signal: AbortSignal, now: () => number): Promise<void> {
117
116
  return new Promise((resolve) => {
118
117
  const done = (): void => {
119
- clearTimeout(timer);
118
+ cancelTimer();
120
119
  signal.removeEventListener('abort', done);
121
120
  resolve();
122
121
  };
123
- const timer = setTimeout(done, ms);
122
+ const cancelTimer = atClockDeadline(now() + ms, now, done);
124
123
  signal.addEventListener('abort', done, { once: true });
125
124
  if (signal.aborted) done();
126
125
  });
@@ -129,11 +128,17 @@ function sleepUnlessAborted(ms: number, signal: AbortSignal): Promise<void> {
129
128
  /** A pre-expiry observation, never an assertion that an open file is a contested lock. */
130
129
  export function captureHangDiagnostics(
131
130
  request: HangCaptureRequest,
132
- source: HangDiagnosticHost = host,
131
+ source: HangDiagnosticHost = {
132
+ platform: process.platform,
133
+ probe: (file, args, timeoutMs, signal) => runProbe(file, args, timeoutMs, signal, undefined, request.awakeNow),
134
+ sleep: (ms, signal) => sleepUnlessAborted(ms, signal, request.awakeNow),
135
+ },
133
136
  ): Promise<HangDiagnostics> {
134
137
  const startedAtMs = Date.now();
135
- const budgetMs = Math.max(0, Math.min(HANG_CAPTURE_MS, request.deadlineMs, request.expiresAtMs - startedAtMs));
136
- const endsAtMs = startedAtMs + budgetMs;
138
+ const { awakeNow, awakeExpiresAtMs } = request;
139
+ const startedAwake = awakeNow();
140
+ const endsAtMs = Math.min(startedAwake + HANG_CAPTURE_MS, awakeExpiresAtMs);
141
+ const budgetMs = Math.max(0, Math.min(HANG_CAPTURE_MS, awakeExpiresAtMs - startedAwake));
137
142
  const controller = new AbortController();
138
143
  const lines = [
139
144
  `Hang snapshot: pid=${request.pid} pgid=${request.pgid ?? 'unknown'} startedAtMs=${startedAtMs}`,
@@ -149,11 +154,11 @@ export function captureHangDiagnostics(
149
154
  const finish = (outcome: HangDiagnostics['outcome']): void => {
150
155
  if (finished) return;
151
156
  finished = true;
152
- clearTimeout(timer);
157
+ cancelTimer();
153
158
  request.signal?.removeEventListener('abort', cancelled);
154
159
  controller.abort();
155
160
  const capturedAtMs = Date.now();
156
- const offsetFromExpiryMs = request.expiresAtMs - capturedAtMs;
161
+ const offsetFromExpiryMs = awakeExpiresAtMs - awakeNow();
157
162
  resolve({
158
163
  startedAtMs,
159
164
  capturedAtMs,
@@ -164,10 +169,12 @@ export function captureHangDiagnostics(
164
169
  text: `${lines.join('\n')}\nHang snapshot outcome=${outcome} capturedAtMs=${capturedAtMs} offsetFromExpiryMs=${offsetFromExpiryMs}\n`,
165
170
  });
166
171
  };
167
- const timer = setTimeout(() => {
168
- lines.push(`Capture incomplete: ${budgetMs}ms deadline reached during ${during()}; no empty-holder conclusion.`);
172
+ const cancelTimer = atClockDeadline(endsAtMs, awakeNow, () => {
173
+ lines.push(
174
+ `Capture incomplete: ${Math.ceil(budgetMs)}ms deadline reached during ${during()}; no empty-holder conclusion.`,
175
+ );
169
176
  finish('incomplete');
170
- }, budgetMs);
177
+ });
171
178
  const cancelled = (): void => {
172
179
  if (finished) return;
173
180
  lines.push(`Capture cancelled during ${during()}; its progress generation is no longer current.`);
@@ -189,7 +196,7 @@ export function captureHangDiagnostics(
189
196
  const result = await source.probe(
190
197
  file,
191
198
  args,
192
- Math.max(0, Math.min(limitMs, endsAtMs - Date.now())),
199
+ Math.max(0, Math.min(limitMs, endsAtMs - awakeNow())),
193
200
  controller.signal,
194
201
  );
195
202
  active.delete(name);
@@ -254,40 +261,45 @@ export function captureHangDiagnostics(
254
261
  }
255
262
  return tree;
256
263
  };
257
- /**
258
- * One-second call stacks of the runner and of its tree's leaf processes, taken at once: a stack names the call a
259
- * process waits in, which the open-file census cannot. A sample sudo refuses, or one that does not answer, is named
260
- * as no stack for that process; stacks never change the holder outcome.
261
- */
262
- const stackSamples = async (tree: readonly ObservedProcess[]): Promise<void> => {
264
+ /** The runner and up to MAX_SAMPLED_LEAVES leaf processes of `tree` (members with no live child), and the leaves past that bound. */
265
+ const leafSamples = (tree: readonly ObservedProcess[]): { sampled: number[]; omitted: number[] } => {
263
266
  const live = tree.filter((entry) => !entry.stat.startsWith('Z'));
264
267
  const parents = new Set(live.map((entry) => entry.ppid));
265
268
  const leaves = live.filter((entry) => entry.pid !== request.pid && !parents.has(entry.pid));
266
- const omitted = leaves.slice(MAX_SAMPLED_LEAVES).map((entry) => entry.pid);
269
+ return {
270
+ sampled: [request.pid, ...leaves.slice(0, MAX_SAMPLED_LEAVES).map((entry) => entry.pid)],
271
+ omitted: leaves.slice(MAX_SAMPLED_LEAVES).map((entry) => entry.pid),
272
+ };
273
+ };
274
+ /**
275
+ * One one-second call stack of `pid`, until the closing census would have less than its own limit left: a stack
276
+ * names the call a process waits in, which the open-file census cannot. A sample sudo refuses, or one that does
277
+ * not answer, is named as no stack for that process; stacks never change the holder outcome.
278
+ */
279
+ const sampleStack = async (pid: number): Promise<void> => {
280
+ const name = `stack sample pid ${pid}`;
281
+ // `-file /dev/stdout` writes the report to the probe's pipe alone, not also to a root-owned file in /tmp.
282
+ const args = ['-n', 'sample', String(pid), String(SAMPLE_SECONDS), '-file', '/dev/stdout'];
283
+ const result = await probe(name, 'sudo', args, endsAtMs - awakeNow() - CENSUS_LIMIT_MS);
284
+ if (finished) return;
285
+ const report = callGraph(result.stdout);
286
+ if (result.status === 'exited' && result.code === 0 && report !== '') {
287
+ lines.push(`${name} (${SAMPLE_SECONDS}s; binary image list omitted):\n${report}`);
288
+ return;
289
+ }
290
+ if (report !== '') lines.push(`Partial ${name} output:\n${report}`);
291
+ lines.push(
292
+ `${name}: no stack; sudo -n sample did not answer, so the call this process waits in is not observed.`,
293
+ );
294
+ };
295
+ /** The runner's and its tree's leaf processes' stacks, taken at once. */
296
+ const stackSamples = async (tree: readonly ObservedProcess[]): Promise<void> => {
297
+ const { sampled, omitted } = leafSamples(tree);
267
298
  if (omitted.length > 0)
268
299
  lines.push(
269
300
  `Stack samples bounded to the runner and ${MAX_SAMPLED_LEAVES} leaf processes; not sampled: pid ${omitted.join(', ')}`,
270
301
  );
271
- const limitMs = endsAtMs - Date.now() - CENSUS_LIMIT_MS;
272
- const sampled = [request.pid, ...leaves.slice(0, MAX_SAMPLED_LEAVES).map((entry) => entry.pid)];
273
- await Promise.all(
274
- sampled.map(async (pid) => {
275
- const name = `stack sample pid ${pid}`;
276
- // `-file /dev/stdout` writes the report to the probe's pipe alone, not also to a root-owned file in /tmp.
277
- const args = ['-n', 'sample', String(pid), String(SAMPLE_SECONDS), '-file', '/dev/stdout'];
278
- const result = await probe(name, 'sudo', args, limitMs);
279
- if (finished) return;
280
- const report = callGraph(result.stdout);
281
- if (result.status === 'exited' && result.code === 0 && report !== '') {
282
- lines.push(`${name} (${SAMPLE_SECONDS}s; binary image list omitted):\n${report}`);
283
- return;
284
- }
285
- if (report !== '') lines.push(`Partial ${name} output:\n${report}`);
286
- lines.push(
287
- `${name}: no stack; sudo -n sample did not answer, so the call this process waits in is not observed.`,
288
- );
289
- }),
290
- );
302
+ await Promise.all(sampled.map(sampleStack));
291
303
  };
292
304
  /**
293
305
  * A process in an uninterruptible kernel wait (state U) is in a file-system or device call no signal ends, and its
@@ -297,7 +309,7 @@ export function captureHangDiagnostics(
297
309
  */
298
310
  const kernelWaits = async (tree: readonly ObservedProcess[]): Promise<void> => {
299
311
  const waiting = tree.filter((entry) => entry.stat.startsWith('U')).slice(0, MAX_SAMPLED_LEAVES);
300
- const limitMs = endsAtMs - Date.now() - CENSUS_LIMIT_MS;
312
+ const limitMs = endsAtMs - awakeNow() - CENSUS_LIMIT_MS;
301
313
  await Promise.all(
302
314
  waiting.map(async ({ pid }) => {
303
315
  const name = `fs_usage pid ${pid}`;
@@ -318,7 +330,7 @@ export function captureHangDiagnostics(
318
330
  const before = await census('ps before');
319
331
  if (finished) return;
320
332
  if (before === null) return finish('incomplete');
321
- const beforeAtMs = Date.now();
333
+ const beforeAtMs = awakeNow();
322
334
  let incomplete = false;
323
335
  let observed = before.length > 0;
324
336
  if (source.platform === 'darwin') {
@@ -370,6 +382,11 @@ export function captureHangDiagnostics(
370
382
  lines.push('Lock holder census incomplete; no lock wait is named.');
371
383
  } else {
372
384
  waits = lockWaits(rows, new Set(before.map((entry) => entry.pid)), openers);
385
+ // A lock wait is open-handle evidence: only the waiter's own stack says whether it blocks in the lock
386
+ // call. Each named waiter the leaf samples left out is sampled the same way, inside the same window.
387
+ const stacked = new Set(leafSamples(before).sampled);
388
+ const waiters = [...new Set(waits.flatMap((wait) => wait.waiters))].filter((pid) => !stacked.has(pid));
389
+ await Promise.all(waiters.slice(0, MAX_SAMPLED_LEAVES).map(sampleStack));
373
390
  }
374
391
  }
375
392
  };
@@ -379,7 +396,7 @@ export function captureHangDiagnostics(
379
396
  lines.push(`Open-file census not provided on ${source.platform}; observations below name processes only.`);
380
397
  lines.push(`Stack samples not provided on ${source.platform}; no blocking call is observed.`);
381
398
  }
382
- const settleMs = Math.min(beforeAtMs + CPU_INTERVAL_MS, endsAtMs - CENSUS_LIMIT_MS) - Date.now();
399
+ const settleMs = Math.min(beforeAtMs + CPU_INTERVAL_MS, endsAtMs - CENSUS_LIMIT_MS) - awakeNow();
383
400
  if (settleMs > 0) {
384
401
  active.add('CPU interval');
385
402
  await source.sleep(settleMs, controller.signal);
@@ -389,7 +406,7 @@ export function captureHangDiagnostics(
389
406
  const after = await census('ps after');
390
407
  if (finished) return;
391
408
  if (after === null) return finish('incomplete');
392
- const afterAtMs = Date.now();
409
+ const afterAtMs = awakeNow();
393
410
  if (
394
411
  before.length !== after.length ||
395
412
  before.some(
@@ -471,6 +488,7 @@ export function runProbe(
471
488
  timeoutMs: number,
472
489
  signal: AbortSignal,
473
490
  environment?: NodeJS.ProcessEnv,
491
+ now: () => number = () => performance.now(),
474
492
  ): Promise<HangProbeResult> {
475
493
  // Keep the package's es2022-compatible event-completion convention.
476
494
  return new Promise((resolve) => {
@@ -486,7 +504,7 @@ export function runProbe(
486
504
  const finish = (result: HangProbeResult): void => {
487
505
  if (settled) return;
488
506
  settled = true;
489
- clearTimeout(timer);
507
+ cancelTimer();
490
508
  signal.removeEventListener('abort', aborted);
491
509
  child.stdout.destroy();
492
510
  child.stderr.destroy();
@@ -508,7 +526,7 @@ export function runProbe(
508
526
  finish({ status, detail: detail + killFailure, stdout, stderr });
509
527
  };
510
528
  const aborted = (): void => stop('capture deadline reached');
511
- const timer = setTimeout(() => stop(`probe exceeded ${timeoutMs}ms`), timeoutMs);
529
+ const cancelTimer = atClockDeadline(now() + timeoutMs, now, () => stop(`probe exceeded ${timeoutMs}ms`));
512
530
  signal.addEventListener('abort', aborted, { once: true });
513
531
  const append = (stream: 'stdout' | 'stderr', chunk: string): void => {
514
532
  bytes += Buffer.byteLength(chunk);
@@ -2,6 +2,7 @@ import type { tags } from 'typia';
2
2
 
3
3
  /** Peer wait classes from specs/cowshed/05_gateway.md, "Deadlines". */
4
4
  export const CONTROL_MS = 5_000;
5
+ export const REAP_MS = 5_000;
5
6
  export const HEARTBEAT_MS = 5_000;
6
7
  export const HEARTBEAT_SILENCE_MS = 15_000;
7
8
 
@@ -45,6 +45,15 @@ export function mbxPool(environment: Readonly<NodeJS.ProcessEnv>): MbxPool | nul
45
45
  return { priority: named ?? 'low' };
46
46
  }
47
47
 
48
+ /**
49
+ * Whether the environment names mbx's pool from inside a hold on it: what a held cowshed job, `mbx reserve`'s command
50
+ * and a reserved leaf's shell see (`MBX_SCHED_DISABLE=1` beside the canonical environment). mbx's test shim still
51
+ * starts every nextest case there, so a run under it may gate its cases (mbx-reservations.ts, `learnHeldGate`).
52
+ */
53
+ export function mbxHold(environment: Readonly<NodeJS.ProcessEnv>): boolean {
54
+ return environment.MBX_SCHEDULER === '1' && !!environment.MBX_CACHE_DIR && environment.MBX_SCHED_DISABLE === '1';
55
+ }
56
+
48
57
  /**
49
58
  * How a leaf enters the pool once. A Cargo leaf that builds runs through mbx, which admits its compilers and supported
50
59
  * test work itself, so wrapping it would charge it twice. Any other leaf reserves for its whole run, asking its Nx
@@ -63,6 +72,11 @@ export type MbxLeaf =
63
72
  readonly kind: 'reserve';
64
73
  readonly priority: MbxPriority;
65
74
  readonly demand: ReservedDemand;
75
+ /**
76
+ * Whether its runner starts every thread of its ask and each case takes a core at the leaf's gate (case-gate.ts):
77
+ * a nextest leaf of a run whose mbx gates its cases. Otherwise its runner is sized to its grant.
78
+ */
79
+ readonly gated: boolean;
66
80
  };
67
81
 
68
82
  /** A leaf mbx's pool reserves for. */
@@ -83,11 +97,12 @@ export function runsPrebuiltNextest(command: string): boolean {
83
97
  * kept every reservation queued behind it waiting while the pool drained. On 2026-10-31 a downstream gate ran 4.9 of 18
84
98
  * cores busy while 25 of its tasks queued a minute or more, and a gate's Go test leaf waited 401 s for all 18 cores.
85
99
  */
86
- export function mbxLeaf(pool: MbxPool, demand: Demand, command: string): MbxLeaf | null {
100
+ export function mbxLeaf(pool: MbxPool, demand: Demand, command: string, testGate: boolean): MbxLeaf | null {
87
101
  const reserve = (reserved: ReservedDemand): MbxLeaf => ({
88
102
  kind: 'reserve',
89
103
  priority: pool.priority,
90
104
  demand: reserved,
105
+ gated: testGate && reserved.kind === 'nextest',
91
106
  });
92
107
  switch (demand.kind) {
93
108
  case 'vitest':
@@ -108,11 +123,11 @@ export function mbxLeaf(pool: MbxPool, demand: Demand, command: string): MbxLeaf
108
123
  /**
109
124
  * Whether a run of `command` under mbx's pool is a Cargo leaf: the one leaf whose admission wait the run reads from
110
125
  * mbx's activity report and its waiters' native lineage (mbx-queue.ts; specs/cowshed/11_shell.md, "Cargo test-runner
111
- * boundaries"), so the only run that loads cowshed's process-lineage entry and its addon. The command alone decides
112
- * it: the pool's priority, the core count and the environment size a leaf, never its kind.
126
+ * boundaries"), through cowshed's process-lineage entry. The command alone decides it: the pool's priority, the core
127
+ * count and the environment size a leaf, never its kind.
113
128
  */
114
129
  export function runsAsCargoLeaf(command: string): boolean {
115
- return mbxLeaf({ priority: 'low' }, runnerDemand(command, undefined, 1, {}), command)?.kind === 'cargo';
130
+ return mbxLeaf({ priority: 'low' }, runnerDemand(command, undefined, 1, {}), command, false)?.kind === 'cargo';
116
131
  }
117
132
 
118
133
  /**
@@ -188,7 +203,8 @@ export const MBX_GRANT_FD = 5;
188
203
 
189
204
  /**
190
205
  * `command` as the shell runs it under its leaf's admission, or as it is for a leaf in no pool (`null`): a reserved
191
- * leaf asks for its runner's count (`reservation`).
206
+ * leaf asks for its runner's count (`reservation`). A reserved leaf's fixture is never part of `command`
207
+ * (`withReservationAfterFixture`).
192
208
  */
193
209
  export function withMbxLeaf(command: string, leaf: MbxLeaf | null): string {
194
210
  if (leaf === null || leaf.kind === 'cargo') {
@@ -197,11 +213,22 @@ export function withMbxLeaf(command: string, leaf: MbxLeaf | null): string {
197
213
  return `${WANT_ENV}=${leaf.demand.want} && ${reservation(command, leaf)}`;
198
214
  }
199
215
 
216
+ /**
217
+ * A reserved leaf with a `fixture`: the fixture runs first, outside the reservation, as pooled work like any other
218
+ * build, and the leaf asks for its runner's count only once it has exited 0 (specs/cowshed/05_gateway.md, "A leaf's
219
+ * build precedes its reservation"). A reservation holds cores only while its command can use them: a fixture inside
220
+ * one waits on Cargo's build-directory lock holding its grant, and when reserved leaves hold every core the lock's
221
+ * holder, a pooled build whose compiler waits for one, never gets it. A fixture that fails fails the run before
222
+ * anything is reserved.
223
+ */
224
+ export function withReservationAfterFixture(fixture: string, command: string, leaf: ReservedLeaf): string {
225
+ return `${fixture} && ${withMbxLeaf(command, leaf)}`;
226
+ }
200
227
  /**
201
228
  * A reserved leaf whose `fixture` segment ends by printing one line per unit its `command` runs (`unitsCommand`): the
202
- * segment runs first, outside the reservation, since a build's compilers are admitted by mbx one by one, and the leaf
203
- * asks for its units, at most its runner's count and at least one. A segment that fails fails the run before anything
204
- * is reserved.
229
+ * segment runs first, outside the reservation as every fixture does (`withReservationAfterFixture`), and the leaf asks
230
+ * for its units, at most its runner's count and at least one. A segment that fails fails the run before anything is
231
+ * reserved.
205
232
  */
206
233
  export function withCountedReservation(fixture: string, command: string, leaf: ReservedLeaf): string {
207
234
  const most = leaf.demand.want;
@@ -222,7 +249,7 @@ function reservation(command: string, leaf: ReservedLeaf): string {
222
249
  const said = `printf 'cowshed: mbx reserve running cpus=%s asked=%s\\n' "$${CPU_TOKENS_ENV}" "$${WANT_ENV}" >&2; `;
223
250
  const ask = `printf '%s\\n' "$${WANT_ENV}" >&${MBX_GRANT_FD} && IFS= read -r ${GRANTED_CPUS_ENV} <&${MBX_GRANT_FD} && exec ${MBX_GRANT_FD}>&-`;
224
251
  const held = `export ${GRANTED_CPUS_ENV} MBX_SCHED_DISABLE=1 MBX_SCHED_DIR=`;
225
- return `export ${WANT_ENV} && ${ask} && ${held} && eval ${shellQuote(`${grantedSizing(leaf.demand)}${said}${command}`)}`;
252
+ return `export ${WANT_ENV} && ${ask} && ${held} && eval ${shellQuote(`${grantedSizing(leaf.demand, leaf.gated)}${said}${command}`)}`;
226
253
  }
227
254
 
228
255
  /**