@smoothbricks/nx-plugin 0.7.18 → 0.7.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +160 -99
- package/dist/.build-inputs.json +35 -29
- package/dist/built-output-packages.js +1 -1
- package/dist/cargo-closure-input.js +1 -1
- package/dist/cargo-source-hash.d.ts +3 -21
- package/dist/cargo-source-hash.d.ts.map +1 -1
- package/dist/cargo-source-hash.js +4 -91
- package/dist/cargo-toolchain-paths.d.ts +2 -2
- package/dist/cargo-toolchain-paths.js +2 -2
- package/dist/cargo-workspace.d.ts.map +1 -1
- package/dist/cargo-workspace.js +2 -1
- package/dist/executors/bounded-exec/admission-queue.d.ts +3 -1
- package/dist/executors/bounded-exec/admission-queue.d.ts.map +1 -1
- package/dist/executors/bounded-exec/admission-queue.js +9 -10
- package/dist/executors/bounded-exec/awake-clock.d.ts +5 -0
- package/dist/executors/bounded-exec/awake-clock.d.ts.map +1 -0
- package/dist/executors/bounded-exec/awake-clock.js +22 -0
- package/dist/executors/bounded-exec/bounds.d.ts +7 -5
- package/dist/executors/bounded-exec/bounds.d.ts.map +1 -1
- package/dist/executors/bounded-exec/bounds.js +14 -11
- package/dist/executors/bounded-exec/case-gate.d.ts +114 -0
- package/dist/executors/bounded-exec/case-gate.d.ts.map +1 -0
- package/dist/executors/bounded-exec/case-gate.js +342 -0
- package/dist/executors/bounded-exec/cpu-pool-thread.d.ts +1 -0
- package/dist/executors/bounded-exec/cpu-pool-thread.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool-thread.js +2 -2
- package/dist/executors/bounded-exec/cpu-pool.d.ts +28 -1
- package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool.js +102 -16
- package/dist/executors/bounded-exec/endpoint-life.d.ts +146 -0
- package/dist/executors/bounded-exec/endpoint-life.d.ts.map +1 -0
- package/dist/executors/bounded-exec/endpoint-life.js +228 -0
- package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
- package/dist/executors/bounded-exec/executor.js +207 -69
- package/dist/executors/bounded-exec/gateway-lease.d.ts +1 -1
- package/dist/executors/bounded-exec/gateway-lease.d.ts.map +1 -1
- package/dist/executors/bounded-exec/gateway-lease.js +8 -1
- package/dist/executors/bounded-exec/go-run-summary.d.ts +51 -0
- package/dist/executors/bounded-exec/go-run-summary.d.ts.map +1 -0
- package/dist/executors/bounded-exec/go-run-summary.js +98 -0
- package/dist/executors/bounded-exec/hang-diagnostics.d.ts +3 -3
- package/dist/executors/bounded-exec/hang-diagnostics.d.ts.map +1 -1
- package/dist/executors/bounded-exec/hang-diagnostics.js +66 -48
- package/dist/executors/bounded-exec/lease-wait.d.ts +1 -0
- package/dist/executors/bounded-exec/lease-wait.d.ts.map +1 -1
- package/dist/executors/bounded-exec/lease-wait.js +1 -0
- package/dist/executors/bounded-exec/mbx-pool.d.ts +28 -7
- package/dist/executors/bounded-exec/mbx-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/mbx-pool.js +30 -9
- package/dist/executors/bounded-exec/mbx-reservations.d.ts +8 -5
- package/dist/executors/bounded-exec/mbx-reservations.d.ts.map +1 -1
- package/dist/executors/bounded-exec/mbx-reservations.js +187 -283
- package/dist/executors/bounded-exec/parent-grant.d.ts +2 -0
- package/dist/executors/bounded-exec/parent-grant.d.ts.map +1 -1
- package/dist/executors/bounded-exec/parent-grant.js +1 -0
- package/dist/executors/bounded-exec/pool-admission.d.ts +29 -3
- package/dist/executors/bounded-exec/pool-admission.d.ts.map +1 -1
- package/dist/executors/bounded-exec/pool-admission.js +47 -1
- package/dist/executors/bounded-exec/progress.d.ts +3 -0
- package/dist/executors/bounded-exec/progress.d.ts.map +1 -1
- package/dist/executors/bounded-exec/progress.js +31 -1
- package/dist/executors/bounded-exec/reservation-connection.d.ts +90 -0
- package/dist/executors/bounded-exec/reservation-connection.d.ts.map +1 -0
- package/dist/executors/bounded-exec/reservation-connection.js +248 -0
- package/dist/executors/bounded-exec/run-cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/run-cpu-pool.js +3 -0
- package/dist/executors/bounded-exec/runner-sizing.d.ts +9 -5
- package/dist/executors/bounded-exec/runner-sizing.d.ts.map +1 -1
- package/dist/executors/bounded-exec/runner-sizing.js +13 -9
- package/dist/executors/bounded-exec/schema.d.ts +2 -2
- package/dist/executors/bounded-exec/suite-minimum.d.ts +21 -0
- package/dist/executors/bounded-exec/suite-minimum.d.ts.map +1 -0
- package/dist/executors/bounded-exec/suite-minimum.js +85 -0
- package/dist/executors/bounded-exec/test-result-writer.d.ts +2 -0
- package/dist/executors/bounded-exec/test-result-writer.d.ts.map +1 -1
- package/dist/executors/bounded-exec/test-result-writer.js +1 -0
- package/dist/executors/bounded-exec/test-results.d.ts +5 -0
- package/dist/executors/bounded-exec/test-results.d.ts.map +1 -1
- package/dist/executors/bounded-exec/test-results.js +2 -2
- package/dist/executors/bounded-exec/verdict.d.ts +26 -4
- package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
- package/dist/executors/bounded-exec/verdict.js +26 -5
- package/dist/executors/cargo-zigbuild/executor.d.ts +10 -8
- package/dist/executors/cargo-zigbuild/executor.d.ts.map +1 -1
- package/dist/executors/cargo-zigbuild/executor.js +32 -19
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +105 -73
- package/dist/managed-files/ci-workflow.d.ts +4 -1
- package/dist/managed-files/ci-workflow.d.ts.map +1 -1
- package/dist/managed-files/ci-workflow.js +9 -14
- package/dist/managed-files/files.d.ts +3 -3
- package/dist/managed-files/files.js +3 -3
- package/dist/task-results.js +2 -1
- package/dist/workspace-config-policy.d.ts.map +1 -1
- package/dist/workspace-config-policy.js +135 -42
- package/executors.json +2 -0
- package/managed/raw/envrc +24 -0
- package/managed/raw/patches/nx@24.0.3.patch +1179 -133
- package/managed/raw/patches/ttsc@0.37.1.patch +1055 -0
- package/managed/raw/tooling/direnv/developer-links.ts +105 -11
- package/managed/raw/tooling/direnv/devenv.smoo.nix +20 -19
- package/package.json +67 -11
- package/runtime/nx-publish/.build-inputs.json +2 -2
- package/runtime/nx-publish/cache_adapter.js +293 -3
- package/runtime/nx-publish/nx_publish.js +36 -14
- package/src/built-output-packages.ts +1 -1
- package/src/cargo-closure-input.ts +1 -1
- package/src/cargo-source-hash.ts +16 -132
- package/src/cargo-toolchain-paths.ts +2 -2
- package/src/cargo-workspace.ts +1 -2
- package/src/executors/bounded-exec/admission-queue.ts +11 -10
- package/src/executors/bounded-exec/awake-clock.ts +20 -0
- package/src/executors/bounded-exec/bounds.ts +28 -15
- package/src/executors/bounded-exec/case-gate.ts +365 -0
- package/src/executors/bounded-exec/cpu-pool-thread.ts +3 -3
- package/src/executors/bounded-exec/cpu-pool.ts +139 -12
- package/src/executors/bounded-exec/endpoint-life.ts +279 -0
- package/src/executors/bounded-exec/executor.ts +280 -106
- package/src/executors/bounded-exec/gateway-lease.ts +6 -0
- package/src/executors/bounded-exec/go-run-summary.ts +137 -0
- package/src/executors/bounded-exec/hang-diagnostics.ts +70 -52
- package/src/executors/bounded-exec/lease-wait.ts +1 -0
- package/src/executors/bounded-exec/mbx-pool.ts +36 -9
- package/src/executors/bounded-exec/mbx-reservations.ts +213 -182
- package/src/executors/bounded-exec/parent-grant.ts +3 -0
- package/src/executors/bounded-exec/pool-admission.ts +59 -2
- package/src/executors/bounded-exec/progress.ts +37 -2
- package/src/executors/bounded-exec/reservation-connection.ts +221 -0
- package/src/executors/bounded-exec/run-cpu-pool.ts +2 -0
- package/src/executors/bounded-exec/runner-sizing.ts +13 -8
- package/src/executors/bounded-exec/schema.json +1 -1
- package/src/executors/bounded-exec/schema.ts +2 -2
- package/src/executors/bounded-exec/suite-minimum.ts +81 -0
- package/src/executors/bounded-exec/test-result-writer.ts +3 -0
- package/src/executors/bounded-exec/test-results.ts +5 -0
- package/src/executors/bounded-exec/verdict.ts +39 -6
- package/src/executors/cargo-zigbuild/executor.ts +30 -19
- package/src/index.ts +112 -78
- package/src/managed-files/ci-workflow.ts +13 -17
- package/src/managed-files/files.ts +3 -3
- package/src/workspace-config-policy.ts +161 -43
- package/test-timings.json +157 -147
- package/managed/raw/patches/ttsc@0.36.3.patch +0 -1660
|
@@ -61,6 +61,7 @@ export function holdLease(
|
|
|
61
61
|
what: string,
|
|
62
62
|
bounds: LeaseBounds | CpuLeaseWait,
|
|
63
63
|
onQueued?: () => void,
|
|
64
|
+
withdrawn?: AbortSignal,
|
|
64
65
|
): Promise<HeldLease> {
|
|
65
66
|
// `Promise.withResolvers` would read better but needs lib es2024; this package inherits lib
|
|
66
67
|
// es2022 from tsconfig.base.json.
|
|
@@ -231,5 +232,10 @@ export function holdLease(
|
|
|
231
232
|
socketClosed = true;
|
|
232
233
|
unleased(`the lease connection for ${what} at ${socket} closed`);
|
|
233
234
|
});
|
|
235
|
+
if (withdrawn !== undefined) {
|
|
236
|
+
const withdraw = (): void => unleased(`the request for ${what} at ${socket} was withdrawn`);
|
|
237
|
+
if (withdrawn.aborted) withdraw();
|
|
238
|
+
else withdrawn.addEventListener('abort', withdraw, { once: true });
|
|
239
|
+
}
|
|
234
240
|
return promise;
|
|
235
241
|
}
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
import { spawnSync } from 'node:child_process';
|
|
2
|
+
import { readFileSync } from 'node:fs';
|
|
3
|
+
import { join, relative, sep } from 'node:path';
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* A Go run by its top-level tests. `go test -json` reports every event and sums none, and a consumer that keeps only
|
|
7
|
+
* part of a long output cannot count it, so the run that read every event states the sum where its output ends.
|
|
8
|
+
*/
|
|
9
|
+
export interface GoRunSummary {
|
|
10
|
+
readonly passed: number;
|
|
11
|
+
readonly failed: number;
|
|
12
|
+
readonly skipped: number;
|
|
13
|
+
readonly packages: number;
|
|
14
|
+
readonly failedTests: readonly GoFailedTest[];
|
|
15
|
+
/** Packages that failed with no failed test of their own: a build that did not compile, a `TestMain` that exited. */
|
|
16
|
+
readonly failedPackages: readonly string[];
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export interface GoFailedTest {
|
|
20
|
+
readonly package: string;
|
|
21
|
+
readonly test: string;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/** One package as `go list -f GO_LIST_FORMAT` prints it: its directory and its in-package and external test files. */
|
|
25
|
+
export interface GoPackageFiles {
|
|
26
|
+
readonly package: string;
|
|
27
|
+
readonly directory: string;
|
|
28
|
+
readonly files: readonly string[];
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export const GO_LIST_FORMAT = '{{.ImportPath}}\t{{.Dir}}\t{{join .TestGoFiles ","}}\t{{join .XTestGoFiles ","}}';
|
|
32
|
+
|
|
33
|
+
export function parseGoList(stdout: string): GoPackageFiles[] {
|
|
34
|
+
const packages: GoPackageFiles[] = [];
|
|
35
|
+
for (const line of stdout.split('\n')) {
|
|
36
|
+
const [name, directory, own, external] = line.split('\t');
|
|
37
|
+
if (name === undefined || name === '' || directory === undefined || directory === '') continue;
|
|
38
|
+
packages.push({
|
|
39
|
+
package: name,
|
|
40
|
+
directory,
|
|
41
|
+
files: [own ?? '', external ?? ''].flatMap((list) => (list === '' ? [] : list.split(','))),
|
|
42
|
+
});
|
|
43
|
+
}
|
|
44
|
+
return packages;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* The file of a package that declares the top-level function `test`, and its line. Go admits one such function per
|
|
49
|
+
* test package, and a directory holds two test packages (in-package and external): a name both declare has no one
|
|
50
|
+
* file, and neither does a name none declares.
|
|
51
|
+
*/
|
|
52
|
+
export function declarationOf(
|
|
53
|
+
test: string,
|
|
54
|
+
files: readonly { readonly path: string; readonly text: string }[],
|
|
55
|
+
): { path: string; line: number } | null {
|
|
56
|
+
const declares = new RegExp(`^func\\s+${test.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\s*\\(`);
|
|
57
|
+
let found: { path: string; line: number } | null = null;
|
|
58
|
+
for (const file of files) {
|
|
59
|
+
const line = file.text.split('\n').findIndex((text) => declares.test(text));
|
|
60
|
+
if (line === -1) continue;
|
|
61
|
+
if (found !== null) return null;
|
|
62
|
+
found = { path: file.path, line: line + 1 };
|
|
63
|
+
}
|
|
64
|
+
return found;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/** Each failed test whose declaration is found, as `<workspace path>:<line>`, keyed `<package>::<test>`. */
|
|
68
|
+
export function goDeclarations(
|
|
69
|
+
failed: readonly GoFailedTest[],
|
|
70
|
+
root: string,
|
|
71
|
+
listed: readonly GoPackageFiles[],
|
|
72
|
+
read: (path: string) => string | null,
|
|
73
|
+
): Map<string, string> {
|
|
74
|
+
const declared = new Map<string, string>();
|
|
75
|
+
for (const { package: name, test } of failed) {
|
|
76
|
+
const found = listed.find((entry) => entry.package === name);
|
|
77
|
+
if (found === undefined) continue;
|
|
78
|
+
const files = found.files.flatMap((file) => {
|
|
79
|
+
const path = join(found.directory, file);
|
|
80
|
+
const text = read(path);
|
|
81
|
+
return text === null ? [] : [{ path, text }];
|
|
82
|
+
});
|
|
83
|
+
const declaration = declarationOf(test, files);
|
|
84
|
+
if (declaration === null) continue;
|
|
85
|
+
declared.set(`${name}::${test}`, `${relative(root, declaration.path).split(sep).join('/')}:${declaration.line}`);
|
|
86
|
+
}
|
|
87
|
+
return declared;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* The lines a Go run's output ends with. A failed test is named at its declaration, the identity a reader matches a
|
|
92
|
+
* source file by; one whose declaration was not found keeps its package, which no reader takes for a file.
|
|
93
|
+
*/
|
|
94
|
+
export function goRunSummaryText(summary: GoRunSummary, declared: ReadonlyMap<string, string>): string {
|
|
95
|
+
const lines = [
|
|
96
|
+
`go test summary: ${summary.passed} passed, ${summary.failed} failed, ${summary.skipped} skipped, ${summary.packages} packages`,
|
|
97
|
+
...summary.failedTests.map(
|
|
98
|
+
({ package: name, test }) => `go test failed: ${declared.get(`${name}::${test}`) ?? name} ${test}`,
|
|
99
|
+
),
|
|
100
|
+
...summary.failedPackages.map((name) => `go test package failed: ${name}`),
|
|
101
|
+
];
|
|
102
|
+
return `${lines.join('\n')}\n`;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/** How long `go list` may take to name the failed packages' directories and test files. */
|
|
106
|
+
const GO_LIST_MS = 60_000;
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* The declarations of a run's failed tests, asked of the toolchain that ran them: `go list` names each failed package's
|
|
110
|
+
* directory and test files, and the file that declares the test is found among them. A list that does not answer
|
|
111
|
+
* leaves every test at its package, which the summary then prints.
|
|
112
|
+
*/
|
|
113
|
+
export function declaredGoTests(
|
|
114
|
+
failed: readonly GoFailedTest[],
|
|
115
|
+
cwd: string,
|
|
116
|
+
root: string,
|
|
117
|
+
env: NodeJS.ProcessEnv,
|
|
118
|
+
): Map<string, string> {
|
|
119
|
+
if (failed.length === 0) return new Map();
|
|
120
|
+
const packages = [...new Set(failed.map((test) => test.package))];
|
|
121
|
+
const listed = spawnSync('go', ['list', '-f', GO_LIST_FORMAT, ...packages], {
|
|
122
|
+
cwd,
|
|
123
|
+
env,
|
|
124
|
+
encoding: 'utf8',
|
|
125
|
+
timeout: GO_LIST_MS,
|
|
126
|
+
});
|
|
127
|
+
if (listed.status !== 0) return new Map();
|
|
128
|
+
return goDeclarations(failed, root, parseGoList(listed.stdout), (path) => {
|
|
129
|
+
try {
|
|
130
|
+
return readFileSync(path, 'utf8');
|
|
131
|
+
} catch (error) {
|
|
132
|
+
// A listed file that cannot be read declares nothing here; any other failure is a bug to see.
|
|
133
|
+
if (error instanceof Error && 'code' in error) return null;
|
|
134
|
+
throw error;
|
|
135
|
+
}
|
|
136
|
+
});
|
|
137
|
+
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { spawn } from 'node:child_process';
|
|
2
2
|
import { describeTeardown, killProcessGroupNow, TEARDOWN_PATIENCE } from '../../process-group.js';
|
|
3
3
|
import { type ProcessEntry, withDescendants } from '../../testing.js';
|
|
4
|
+
import { atClockDeadline } from './awake-clock.js';
|
|
4
5
|
import {
|
|
5
6
|
CPU_RISING_CLASS,
|
|
6
7
|
type CpuRise,
|
|
@@ -37,9 +38,9 @@ const PS_FIELDS = 'pid=,ppid=,pgid=,stat=,%cpu=,etime=,cputime=,lstart=,command=
|
|
|
37
38
|
const CPU_INTERVAL_MS = 5_000;
|
|
38
39
|
/**
|
|
39
40
|
* Stack samples: `sample` suspends a process every millisecond for one second and reports its threads' call graph. The
|
|
40
|
-
* runner and at most this many leaf processes are sampled, all at once
|
|
41
|
-
*
|
|
42
|
-
* capture all the same.
|
|
41
|
+
* runner and at most this many leaf processes are sampled, all at once, and then at most this many named lock waiters
|
|
42
|
+
* those left out. `-mayDie` is not passed: it reads symbols before sampling and measured about a second slower under
|
|
43
|
+
* load, and a process that exits mid-sample is a raced capture all the same.
|
|
43
44
|
*/
|
|
44
45
|
const MAX_SAMPLED_LEAVES = 4;
|
|
45
46
|
const SAMPLE_SECONDS = 1;
|
|
@@ -87,8 +88,8 @@ export interface HangDiagnostics {
|
|
|
87
88
|
export interface HangCaptureRequest {
|
|
88
89
|
pid: number;
|
|
89
90
|
pgid: number | null;
|
|
90
|
-
|
|
91
|
-
|
|
91
|
+
awakeExpiresAtMs: number;
|
|
92
|
+
awakeNow(): number;
|
|
92
93
|
signal?: AbortSignal;
|
|
93
94
|
}
|
|
94
95
|
|
|
@@ -111,16 +112,14 @@ interface ObservedProcess extends ProcessEntry {
|
|
|
111
112
|
raw: string;
|
|
112
113
|
}
|
|
113
114
|
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
function sleepUnlessAborted(ms: number, signal: AbortSignal): Promise<void> {
|
|
115
|
+
function sleepUnlessAborted(ms: number, signal: AbortSignal, now: () => number): Promise<void> {
|
|
117
116
|
return new Promise((resolve) => {
|
|
118
117
|
const done = (): void => {
|
|
119
|
-
|
|
118
|
+
cancelTimer();
|
|
120
119
|
signal.removeEventListener('abort', done);
|
|
121
120
|
resolve();
|
|
122
121
|
};
|
|
123
|
-
const
|
|
122
|
+
const cancelTimer = atClockDeadline(now() + ms, now, done);
|
|
124
123
|
signal.addEventListener('abort', done, { once: true });
|
|
125
124
|
if (signal.aborted) done();
|
|
126
125
|
});
|
|
@@ -129,11 +128,17 @@ function sleepUnlessAborted(ms: number, signal: AbortSignal): Promise<void> {
|
|
|
129
128
|
/** A pre-expiry observation, never an assertion that an open file is a contested lock. */
|
|
130
129
|
export function captureHangDiagnostics(
|
|
131
130
|
request: HangCaptureRequest,
|
|
132
|
-
source: HangDiagnosticHost =
|
|
131
|
+
source: HangDiagnosticHost = {
|
|
132
|
+
platform: process.platform,
|
|
133
|
+
probe: (file, args, timeoutMs, signal) => runProbe(file, args, timeoutMs, signal, undefined, request.awakeNow),
|
|
134
|
+
sleep: (ms, signal) => sleepUnlessAborted(ms, signal, request.awakeNow),
|
|
135
|
+
},
|
|
133
136
|
): Promise<HangDiagnostics> {
|
|
134
137
|
const startedAtMs = Date.now();
|
|
135
|
-
const
|
|
136
|
-
const
|
|
138
|
+
const { awakeNow, awakeExpiresAtMs } = request;
|
|
139
|
+
const startedAwake = awakeNow();
|
|
140
|
+
const endsAtMs = Math.min(startedAwake + HANG_CAPTURE_MS, awakeExpiresAtMs);
|
|
141
|
+
const budgetMs = Math.max(0, Math.min(HANG_CAPTURE_MS, awakeExpiresAtMs - startedAwake));
|
|
137
142
|
const controller = new AbortController();
|
|
138
143
|
const lines = [
|
|
139
144
|
`Hang snapshot: pid=${request.pid} pgid=${request.pgid ?? 'unknown'} startedAtMs=${startedAtMs}`,
|
|
@@ -149,11 +154,11 @@ export function captureHangDiagnostics(
|
|
|
149
154
|
const finish = (outcome: HangDiagnostics['outcome']): void => {
|
|
150
155
|
if (finished) return;
|
|
151
156
|
finished = true;
|
|
152
|
-
|
|
157
|
+
cancelTimer();
|
|
153
158
|
request.signal?.removeEventListener('abort', cancelled);
|
|
154
159
|
controller.abort();
|
|
155
160
|
const capturedAtMs = Date.now();
|
|
156
|
-
const offsetFromExpiryMs =
|
|
161
|
+
const offsetFromExpiryMs = awakeExpiresAtMs - awakeNow();
|
|
157
162
|
resolve({
|
|
158
163
|
startedAtMs,
|
|
159
164
|
capturedAtMs,
|
|
@@ -164,10 +169,12 @@ export function captureHangDiagnostics(
|
|
|
164
169
|
text: `${lines.join('\n')}\nHang snapshot outcome=${outcome} capturedAtMs=${capturedAtMs} offsetFromExpiryMs=${offsetFromExpiryMs}\n`,
|
|
165
170
|
});
|
|
166
171
|
};
|
|
167
|
-
const
|
|
168
|
-
lines.push(
|
|
172
|
+
const cancelTimer = atClockDeadline(endsAtMs, awakeNow, () => {
|
|
173
|
+
lines.push(
|
|
174
|
+
`Capture incomplete: ${Math.ceil(budgetMs)}ms deadline reached during ${during()}; no empty-holder conclusion.`,
|
|
175
|
+
);
|
|
169
176
|
finish('incomplete');
|
|
170
|
-
}
|
|
177
|
+
});
|
|
171
178
|
const cancelled = (): void => {
|
|
172
179
|
if (finished) return;
|
|
173
180
|
lines.push(`Capture cancelled during ${during()}; its progress generation is no longer current.`);
|
|
@@ -189,7 +196,7 @@ export function captureHangDiagnostics(
|
|
|
189
196
|
const result = await source.probe(
|
|
190
197
|
file,
|
|
191
198
|
args,
|
|
192
|
-
Math.max(0, Math.min(limitMs, endsAtMs -
|
|
199
|
+
Math.max(0, Math.min(limitMs, endsAtMs - awakeNow())),
|
|
193
200
|
controller.signal,
|
|
194
201
|
);
|
|
195
202
|
active.delete(name);
|
|
@@ -254,40 +261,45 @@ export function captureHangDiagnostics(
|
|
|
254
261
|
}
|
|
255
262
|
return tree;
|
|
256
263
|
};
|
|
257
|
-
/**
|
|
258
|
-
|
|
259
|
-
* process waits in, which the open-file census cannot. A sample sudo refuses, or one that does not answer, is named
|
|
260
|
-
* as no stack for that process; stacks never change the holder outcome.
|
|
261
|
-
*/
|
|
262
|
-
const stackSamples = async (tree: readonly ObservedProcess[]): Promise<void> => {
|
|
264
|
+
/** The runner and up to MAX_SAMPLED_LEAVES leaf processes of `tree` (members with no live child), and the leaves past that bound. */
|
|
265
|
+
const leafSamples = (tree: readonly ObservedProcess[]): { sampled: number[]; omitted: number[] } => {
|
|
263
266
|
const live = tree.filter((entry) => !entry.stat.startsWith('Z'));
|
|
264
267
|
const parents = new Set(live.map((entry) => entry.ppid));
|
|
265
268
|
const leaves = live.filter((entry) => entry.pid !== request.pid && !parents.has(entry.pid));
|
|
266
|
-
|
|
269
|
+
return {
|
|
270
|
+
sampled: [request.pid, ...leaves.slice(0, MAX_SAMPLED_LEAVES).map((entry) => entry.pid)],
|
|
271
|
+
omitted: leaves.slice(MAX_SAMPLED_LEAVES).map((entry) => entry.pid),
|
|
272
|
+
};
|
|
273
|
+
};
|
|
274
|
+
/**
|
|
275
|
+
* One one-second call stack of `pid`, until the closing census would have less than its own limit left: a stack
|
|
276
|
+
* names the call a process waits in, which the open-file census cannot. A sample sudo refuses, or one that does
|
|
277
|
+
* not answer, is named as no stack for that process; stacks never change the holder outcome.
|
|
278
|
+
*/
|
|
279
|
+
const sampleStack = async (pid: number): Promise<void> => {
|
|
280
|
+
const name = `stack sample pid ${pid}`;
|
|
281
|
+
// `-file /dev/stdout` writes the report to the probe's pipe alone, not also to a root-owned file in /tmp.
|
|
282
|
+
const args = ['-n', 'sample', String(pid), String(SAMPLE_SECONDS), '-file', '/dev/stdout'];
|
|
283
|
+
const result = await probe(name, 'sudo', args, endsAtMs - awakeNow() - CENSUS_LIMIT_MS);
|
|
284
|
+
if (finished) return;
|
|
285
|
+
const report = callGraph(result.stdout);
|
|
286
|
+
if (result.status === 'exited' && result.code === 0 && report !== '') {
|
|
287
|
+
lines.push(`${name} (${SAMPLE_SECONDS}s; binary image list omitted):\n${report}`);
|
|
288
|
+
return;
|
|
289
|
+
}
|
|
290
|
+
if (report !== '') lines.push(`Partial ${name} output:\n${report}`);
|
|
291
|
+
lines.push(
|
|
292
|
+
`${name}: no stack; sudo -n sample did not answer, so the call this process waits in is not observed.`,
|
|
293
|
+
);
|
|
294
|
+
};
|
|
295
|
+
/** The runner's and its tree's leaf processes' stacks, taken at once. */
|
|
296
|
+
const stackSamples = async (tree: readonly ObservedProcess[]): Promise<void> => {
|
|
297
|
+
const { sampled, omitted } = leafSamples(tree);
|
|
267
298
|
if (omitted.length > 0)
|
|
268
299
|
lines.push(
|
|
269
300
|
`Stack samples bounded to the runner and ${MAX_SAMPLED_LEAVES} leaf processes; not sampled: pid ${omitted.join(', ')}`,
|
|
270
301
|
);
|
|
271
|
-
|
|
272
|
-
const sampled = [request.pid, ...leaves.slice(0, MAX_SAMPLED_LEAVES).map((entry) => entry.pid)];
|
|
273
|
-
await Promise.all(
|
|
274
|
-
sampled.map(async (pid) => {
|
|
275
|
-
const name = `stack sample pid ${pid}`;
|
|
276
|
-
// `-file /dev/stdout` writes the report to the probe's pipe alone, not also to a root-owned file in /tmp.
|
|
277
|
-
const args = ['-n', 'sample', String(pid), String(SAMPLE_SECONDS), '-file', '/dev/stdout'];
|
|
278
|
-
const result = await probe(name, 'sudo', args, limitMs);
|
|
279
|
-
if (finished) return;
|
|
280
|
-
const report = callGraph(result.stdout);
|
|
281
|
-
if (result.status === 'exited' && result.code === 0 && report !== '') {
|
|
282
|
-
lines.push(`${name} (${SAMPLE_SECONDS}s; binary image list omitted):\n${report}`);
|
|
283
|
-
return;
|
|
284
|
-
}
|
|
285
|
-
if (report !== '') lines.push(`Partial ${name} output:\n${report}`);
|
|
286
|
-
lines.push(
|
|
287
|
-
`${name}: no stack; sudo -n sample did not answer, so the call this process waits in is not observed.`,
|
|
288
|
-
);
|
|
289
|
-
}),
|
|
290
|
-
);
|
|
302
|
+
await Promise.all(sampled.map(sampleStack));
|
|
291
303
|
};
|
|
292
304
|
/**
|
|
293
305
|
* A process in an uninterruptible kernel wait (state U) is in a file-system or device call no signal ends, and its
|
|
@@ -297,7 +309,7 @@ export function captureHangDiagnostics(
|
|
|
297
309
|
*/
|
|
298
310
|
const kernelWaits = async (tree: readonly ObservedProcess[]): Promise<void> => {
|
|
299
311
|
const waiting = tree.filter((entry) => entry.stat.startsWith('U')).slice(0, MAX_SAMPLED_LEAVES);
|
|
300
|
-
const limitMs = endsAtMs -
|
|
312
|
+
const limitMs = endsAtMs - awakeNow() - CENSUS_LIMIT_MS;
|
|
301
313
|
await Promise.all(
|
|
302
314
|
waiting.map(async ({ pid }) => {
|
|
303
315
|
const name = `fs_usage pid ${pid}`;
|
|
@@ -318,7 +330,7 @@ export function captureHangDiagnostics(
|
|
|
318
330
|
const before = await census('ps before');
|
|
319
331
|
if (finished) return;
|
|
320
332
|
if (before === null) return finish('incomplete');
|
|
321
|
-
const beforeAtMs =
|
|
333
|
+
const beforeAtMs = awakeNow();
|
|
322
334
|
let incomplete = false;
|
|
323
335
|
let observed = before.length > 0;
|
|
324
336
|
if (source.platform === 'darwin') {
|
|
@@ -370,6 +382,11 @@ export function captureHangDiagnostics(
|
|
|
370
382
|
lines.push('Lock holder census incomplete; no lock wait is named.');
|
|
371
383
|
} else {
|
|
372
384
|
waits = lockWaits(rows, new Set(before.map((entry) => entry.pid)), openers);
|
|
385
|
+
// A lock wait is open-handle evidence: only the waiter's own stack says whether it blocks in the lock
|
|
386
|
+
// call. Each named waiter the leaf samples left out is sampled the same way, inside the same window.
|
|
387
|
+
const stacked = new Set(leafSamples(before).sampled);
|
|
388
|
+
const waiters = [...new Set(waits.flatMap((wait) => wait.waiters))].filter((pid) => !stacked.has(pid));
|
|
389
|
+
await Promise.all(waiters.slice(0, MAX_SAMPLED_LEAVES).map(sampleStack));
|
|
373
390
|
}
|
|
374
391
|
}
|
|
375
392
|
};
|
|
@@ -379,7 +396,7 @@ export function captureHangDiagnostics(
|
|
|
379
396
|
lines.push(`Open-file census not provided on ${source.platform}; observations below name processes only.`);
|
|
380
397
|
lines.push(`Stack samples not provided on ${source.platform}; no blocking call is observed.`);
|
|
381
398
|
}
|
|
382
|
-
const settleMs = Math.min(beforeAtMs + CPU_INTERVAL_MS, endsAtMs - CENSUS_LIMIT_MS) -
|
|
399
|
+
const settleMs = Math.min(beforeAtMs + CPU_INTERVAL_MS, endsAtMs - CENSUS_LIMIT_MS) - awakeNow();
|
|
383
400
|
if (settleMs > 0) {
|
|
384
401
|
active.add('CPU interval');
|
|
385
402
|
await source.sleep(settleMs, controller.signal);
|
|
@@ -389,7 +406,7 @@ export function captureHangDiagnostics(
|
|
|
389
406
|
const after = await census('ps after');
|
|
390
407
|
if (finished) return;
|
|
391
408
|
if (after === null) return finish('incomplete');
|
|
392
|
-
const afterAtMs =
|
|
409
|
+
const afterAtMs = awakeNow();
|
|
393
410
|
if (
|
|
394
411
|
before.length !== after.length ||
|
|
395
412
|
before.some(
|
|
@@ -471,6 +488,7 @@ export function runProbe(
|
|
|
471
488
|
timeoutMs: number,
|
|
472
489
|
signal: AbortSignal,
|
|
473
490
|
environment?: NodeJS.ProcessEnv,
|
|
491
|
+
now: () => number = () => performance.now(),
|
|
474
492
|
): Promise<HangProbeResult> {
|
|
475
493
|
// Keep the package's es2022-compatible event-completion convention.
|
|
476
494
|
return new Promise((resolve) => {
|
|
@@ -486,7 +504,7 @@ export function runProbe(
|
|
|
486
504
|
const finish = (result: HangProbeResult): void => {
|
|
487
505
|
if (settled) return;
|
|
488
506
|
settled = true;
|
|
489
|
-
|
|
507
|
+
cancelTimer();
|
|
490
508
|
signal.removeEventListener('abort', aborted);
|
|
491
509
|
child.stdout.destroy();
|
|
492
510
|
child.stderr.destroy();
|
|
@@ -508,7 +526,7 @@ export function runProbe(
|
|
|
508
526
|
finish({ status, detail: detail + killFailure, stdout, stderr });
|
|
509
527
|
};
|
|
510
528
|
const aborted = (): void => stop('capture deadline reached');
|
|
511
|
-
const
|
|
529
|
+
const cancelTimer = atClockDeadline(now() + timeoutMs, now, () => stop(`probe exceeded ${timeoutMs}ms`));
|
|
512
530
|
signal.addEventListener('abort', aborted, { once: true });
|
|
513
531
|
const append = (stream: 'stdout' | 'stderr', chunk: string): void => {
|
|
514
532
|
bytes += Buffer.byteLength(chunk);
|
|
@@ -45,6 +45,15 @@ export function mbxPool(environment: Readonly<NodeJS.ProcessEnv>): MbxPool | nul
|
|
|
45
45
|
return { priority: named ?? 'low' };
|
|
46
46
|
}
|
|
47
47
|
|
|
48
|
+
/**
|
|
49
|
+
* Whether the environment names mbx's pool from inside a hold on it: what a held cowshed job, `mbx reserve`'s command
|
|
50
|
+
* and a reserved leaf's shell see (`MBX_SCHED_DISABLE=1` beside the canonical environment). mbx's test shim still
|
|
51
|
+
* starts every nextest case there, so a run under it may gate its cases (mbx-reservations.ts, `learnHeldGate`).
|
|
52
|
+
*/
|
|
53
|
+
export function mbxHold(environment: Readonly<NodeJS.ProcessEnv>): boolean {
|
|
54
|
+
return environment.MBX_SCHEDULER === '1' && !!environment.MBX_CACHE_DIR && environment.MBX_SCHED_DISABLE === '1';
|
|
55
|
+
}
|
|
56
|
+
|
|
48
57
|
/**
|
|
49
58
|
* How a leaf enters the pool once. A Cargo leaf that builds runs through mbx, which admits its compilers and supported
|
|
50
59
|
* test work itself, so wrapping it would charge it twice. Any other leaf reserves for its whole run, asking its Nx
|
|
@@ -63,6 +72,11 @@ export type MbxLeaf =
|
|
|
63
72
|
readonly kind: 'reserve';
|
|
64
73
|
readonly priority: MbxPriority;
|
|
65
74
|
readonly demand: ReservedDemand;
|
|
75
|
+
/**
|
|
76
|
+
* Whether its runner starts every thread of its ask and each case takes a core at the leaf's gate (case-gate.ts):
|
|
77
|
+
* a nextest leaf of a run whose mbx gates its cases. Otherwise its runner is sized to its grant.
|
|
78
|
+
*/
|
|
79
|
+
readonly gated: boolean;
|
|
66
80
|
};
|
|
67
81
|
|
|
68
82
|
/** A leaf mbx's pool reserves for. */
|
|
@@ -83,11 +97,12 @@ export function runsPrebuiltNextest(command: string): boolean {
|
|
|
83
97
|
* kept every reservation queued behind it waiting while the pool drained. On 2026-10-31 a downstream gate ran 4.9 of 18
|
|
84
98
|
* cores busy while 25 of its tasks queued a minute or more, and a gate's Go test leaf waited 401 s for all 18 cores.
|
|
85
99
|
*/
|
|
86
|
-
export function mbxLeaf(pool: MbxPool, demand: Demand, command: string): MbxLeaf | null {
|
|
100
|
+
export function mbxLeaf(pool: MbxPool, demand: Demand, command: string, testGate: boolean): MbxLeaf | null {
|
|
87
101
|
const reserve = (reserved: ReservedDemand): MbxLeaf => ({
|
|
88
102
|
kind: 'reserve',
|
|
89
103
|
priority: pool.priority,
|
|
90
104
|
demand: reserved,
|
|
105
|
+
gated: testGate && reserved.kind === 'nextest',
|
|
91
106
|
});
|
|
92
107
|
switch (demand.kind) {
|
|
93
108
|
case 'vitest':
|
|
@@ -108,11 +123,11 @@ export function mbxLeaf(pool: MbxPool, demand: Demand, command: string): MbxLeaf
|
|
|
108
123
|
/**
|
|
109
124
|
* Whether a run of `command` under mbx's pool is a Cargo leaf: the one leaf whose admission wait the run reads from
|
|
110
125
|
* mbx's activity report and its waiters' native lineage (mbx-queue.ts; specs/cowshed/11_shell.md, "Cargo test-runner
|
|
111
|
-
* boundaries"),
|
|
112
|
-
*
|
|
126
|
+
* boundaries"), through cowshed's process-lineage entry. The command alone decides it: the pool's priority, the core
|
|
127
|
+
* count and the environment size a leaf, never its kind.
|
|
113
128
|
*/
|
|
114
129
|
export function runsAsCargoLeaf(command: string): boolean {
|
|
115
|
-
return mbxLeaf({ priority: 'low' }, runnerDemand(command, undefined, 1, {}), command)?.kind === 'cargo';
|
|
130
|
+
return mbxLeaf({ priority: 'low' }, runnerDemand(command, undefined, 1, {}), command, false)?.kind === 'cargo';
|
|
116
131
|
}
|
|
117
132
|
|
|
118
133
|
/**
|
|
@@ -188,7 +203,8 @@ export const MBX_GRANT_FD = 5;
|
|
|
188
203
|
|
|
189
204
|
/**
|
|
190
205
|
* `command` as the shell runs it under its leaf's admission, or as it is for a leaf in no pool (`null`): a reserved
|
|
191
|
-
* leaf asks for its runner's count (`reservation`).
|
|
206
|
+
* leaf asks for its runner's count (`reservation`). A reserved leaf's fixture is never part of `command`
|
|
207
|
+
* (`withReservationAfterFixture`).
|
|
192
208
|
*/
|
|
193
209
|
export function withMbxLeaf(command: string, leaf: MbxLeaf | null): string {
|
|
194
210
|
if (leaf === null || leaf.kind === 'cargo') {
|
|
@@ -197,11 +213,22 @@ export function withMbxLeaf(command: string, leaf: MbxLeaf | null): string {
|
|
|
197
213
|
return `${WANT_ENV}=${leaf.demand.want} && ${reservation(command, leaf)}`;
|
|
198
214
|
}
|
|
199
215
|
|
|
216
|
+
/**
|
|
217
|
+
* A reserved leaf with a `fixture`: the fixture runs first, outside the reservation, as pooled work like any other
|
|
218
|
+
* build, and the leaf asks for its runner's count only once it has exited 0 (specs/cowshed/05_gateway.md, "A leaf's
|
|
219
|
+
* build precedes its reservation"). A reservation holds cores only while its command can use them: a fixture inside
|
|
220
|
+
* one waits on Cargo's build-directory lock holding its grant, and when reserved leaves hold every core the lock's
|
|
221
|
+
* holder, a pooled build whose compiler waits for one, never gets it. A fixture that fails fails the run before
|
|
222
|
+
* anything is reserved.
|
|
223
|
+
*/
|
|
224
|
+
export function withReservationAfterFixture(fixture: string, command: string, leaf: ReservedLeaf): string {
|
|
225
|
+
return `${fixture} && ${withMbxLeaf(command, leaf)}`;
|
|
226
|
+
}
|
|
200
227
|
/**
|
|
201
228
|
* A reserved leaf whose `fixture` segment ends by printing one line per unit its `command` runs (`unitsCommand`): the
|
|
202
|
-
* segment runs first, outside the reservation
|
|
203
|
-
*
|
|
204
|
-
*
|
|
229
|
+
* segment runs first, outside the reservation as every fixture does (`withReservationAfterFixture`), and the leaf asks
|
|
230
|
+
* for its units, at most its runner's count and at least one. A segment that fails fails the run before anything is
|
|
231
|
+
* reserved.
|
|
205
232
|
*/
|
|
206
233
|
export function withCountedReservation(fixture: string, command: string, leaf: ReservedLeaf): string {
|
|
207
234
|
const most = leaf.demand.want;
|
|
@@ -222,7 +249,7 @@ function reservation(command: string, leaf: ReservedLeaf): string {
|
|
|
222
249
|
const said = `printf 'cowshed: mbx reserve running cpus=%s asked=%s\\n' "$${CPU_TOKENS_ENV}" "$${WANT_ENV}" >&2; `;
|
|
223
250
|
const ask = `printf '%s\\n' "$${WANT_ENV}" >&${MBX_GRANT_FD} && IFS= read -r ${GRANTED_CPUS_ENV} <&${MBX_GRANT_FD} && exec ${MBX_GRANT_FD}>&-`;
|
|
224
251
|
const held = `export ${GRANTED_CPUS_ENV} MBX_SCHED_DISABLE=1 MBX_SCHED_DIR=`;
|
|
225
|
-
return `export ${WANT_ENV} && ${ask} && ${held} && eval ${shellQuote(`${grantedSizing(leaf.demand)}${said}${command}`)}`;
|
|
252
|
+
return `export ${WANT_ENV} && ${ask} && ${held} && eval ${shellQuote(`${grantedSizing(leaf.demand, leaf.gated)}${said}${command}`)}`;
|
|
226
253
|
}
|
|
227
254
|
|
|
228
255
|
/**
|