@smoothbricks/nx-plugin 0.7.17 → 0.7.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +315 -114
- package/dist/.build-inputs.json +47 -38
- package/dist/built-output-packages.js +1 -1
- package/dist/bun-lock-scope.d.ts +128 -20
- package/dist/bun-lock-scope.d.ts.map +1 -1
- package/dist/bun-lock-scope.js +267 -44
- package/dist/cargo-closure-input.d.ts +8 -6
- package/dist/cargo-closure-input.d.ts.map +1 -1
- package/dist/cargo-closure-input.js +15 -12
- package/dist/cargo-source-hash.d.ts +26 -25
- package/dist/cargo-source-hash.d.ts.map +1 -1
- package/dist/cargo-source-hash.js +84 -105
- package/dist/cargo-toolchain-paths.d.ts +2 -2
- package/dist/cargo-toolchain-paths.js +2 -2
- package/dist/cargo-workspace.d.ts +20 -2
- package/dist/cargo-workspace.d.ts.map +1 -1
- package/dist/cargo-workspace.js +43 -22
- package/dist/ensure-built-receipt.d.ts +3 -2
- package/dist/ensure-built-receipt.d.ts.map +1 -1
- package/dist/ensure-built-receipt.js +3 -2
- package/dist/eslint-config-closure.d.ts +8 -3
- package/dist/eslint-config-closure.d.ts.map +1 -1
- package/dist/eslint-config-closure.js +24 -5
- package/dist/executors/bounded-exec/admission-queue.d.ts +3 -1
- package/dist/executors/bounded-exec/admission-queue.d.ts.map +1 -1
- package/dist/executors/bounded-exec/admission-queue.js +9 -10
- package/dist/executors/bounded-exec/awake-clock.d.ts +5 -0
- package/dist/executors/bounded-exec/awake-clock.d.ts.map +1 -0
- package/dist/executors/bounded-exec/awake-clock.js +22 -0
- package/dist/executors/bounded-exec/bounds.d.ts +19 -6
- package/dist/executors/bounded-exec/bounds.d.ts.map +1 -1
- package/dist/executors/bounded-exec/bounds.js +22 -15
- package/dist/executors/bounded-exec/case-gate.d.ts +114 -0
- package/dist/executors/bounded-exec/case-gate.d.ts.map +1 -0
- package/dist/executors/bounded-exec/case-gate.js +342 -0
- package/dist/executors/bounded-exec/cpu-pool-thread.d.ts +1 -0
- package/dist/executors/bounded-exec/cpu-pool-thread.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool-thread.js +2 -2
- package/dist/executors/bounded-exec/cpu-pool.d.ts +28 -1
- package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool.js +102 -16
- package/dist/executors/bounded-exec/cpu-rise.d.ts +73 -0
- package/dist/executors/bounded-exec/cpu-rise.d.ts.map +1 -0
- package/dist/executors/bounded-exec/cpu-rise.js +111 -0
- package/dist/executors/bounded-exec/endpoint-life.d.ts +146 -0
- package/dist/executors/bounded-exec/endpoint-life.d.ts.map +1 -0
- package/dist/executors/bounded-exec/endpoint-life.js +228 -0
- package/dist/executors/bounded-exec/executor.d.ts +1 -1
- package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
- package/dist/executors/bounded-exec/executor.js +256 -86
- package/dist/executors/bounded-exec/gateway-lease.d.ts +1 -1
- package/dist/executors/bounded-exec/gateway-lease.d.ts.map +1 -1
- package/dist/executors/bounded-exec/gateway-lease.js +8 -1
- package/dist/executors/bounded-exec/go-run-summary.d.ts +51 -0
- package/dist/executors/bounded-exec/go-run-summary.d.ts.map +1 -0
- package/dist/executors/bounded-exec/go-run-summary.js +98 -0
- package/dist/executors/bounded-exec/hang-diagnostics.d.ts +11 -3
- package/dist/executors/bounded-exec/hang-diagnostics.d.ts.map +1 -1
- package/dist/executors/bounded-exec/hang-diagnostics.js +125 -47
- package/dist/executors/bounded-exec/lease-wait.d.ts +1 -0
- package/dist/executors/bounded-exec/lease-wait.d.ts.map +1 -1
- package/dist/executors/bounded-exec/lease-wait.js +1 -0
- package/dist/executors/bounded-exec/mbx-pool.d.ts +42 -7
- package/dist/executors/bounded-exec/mbx-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/mbx-pool.js +45 -8
- package/dist/executors/bounded-exec/mbx-reservations.d.ts +12 -6
- package/dist/executors/bounded-exec/mbx-reservations.d.ts.map +1 -1
- package/dist/executors/bounded-exec/mbx-reservations.js +190 -284
- package/dist/executors/bounded-exec/parent-grant.d.ts +2 -0
- package/dist/executors/bounded-exec/parent-grant.d.ts.map +1 -1
- package/dist/executors/bounded-exec/parent-grant.js +1 -0
- package/dist/executors/bounded-exec/pool-admission.d.ts +29 -3
- package/dist/executors/bounded-exec/pool-admission.d.ts.map +1 -1
- package/dist/executors/bounded-exec/pool-admission.js +47 -1
- package/dist/executors/bounded-exec/progress-channel.d.ts +38 -0
- package/dist/executors/bounded-exec/progress-channel.d.ts.map +1 -0
- package/dist/executors/bounded-exec/progress-channel.js +113 -0
- package/dist/executors/bounded-exec/progress.d.ts +5 -1
- package/dist/executors/bounded-exec/progress.d.ts.map +1 -1
- package/dist/executors/bounded-exec/progress.js +31 -1
- package/dist/executors/bounded-exec/ram-temp.d.ts +24 -1
- package/dist/executors/bounded-exec/ram-temp.d.ts.map +1 -1
- package/dist/executors/bounded-exec/ram-temp.js +123 -16
- package/dist/executors/bounded-exec/reservation-connection.d.ts +90 -0
- package/dist/executors/bounded-exec/reservation-connection.d.ts.map +1 -0
- package/dist/executors/bounded-exec/reservation-connection.js +248 -0
- package/dist/executors/bounded-exec/run-cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/run-cpu-pool.js +3 -0
- package/dist/executors/bounded-exec/runner-sizing.d.ts +9 -5
- package/dist/executors/bounded-exec/runner-sizing.d.ts.map +1 -1
- package/dist/executors/bounded-exec/runner-sizing.js +13 -9
- package/dist/executors/bounded-exec/schema.d.ts +7 -2
- package/dist/executors/bounded-exec/schema.d.ts.map +1 -1
- package/dist/executors/bounded-exec/schema.js +17 -1
- package/dist/executors/bounded-exec/suite-minimum.d.ts +21 -0
- package/dist/executors/bounded-exec/suite-minimum.d.ts.map +1 -0
- package/dist/executors/bounded-exec/suite-minimum.js +85 -0
- package/dist/executors/bounded-exec/test-result-writer.d.ts +2 -0
- package/dist/executors/bounded-exec/test-result-writer.d.ts.map +1 -1
- package/dist/executors/bounded-exec/test-result-writer.js +1 -0
- package/dist/executors/bounded-exec/test-results.d.ts +5 -0
- package/dist/executors/bounded-exec/test-results.d.ts.map +1 -1
- package/dist/executors/bounded-exec/test-results.js +2 -2
- package/dist/executors/bounded-exec/verdict.d.ts +39 -10
- package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
- package/dist/executors/bounded-exec/verdict.js +60 -14
- package/dist/executors/cargo-zigbuild/executor.d.ts +10 -8
- package/dist/executors/cargo-zigbuild/executor.d.ts.map +1 -1
- package/dist/executors/cargo-zigbuild/executor.js +32 -19
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +294 -152
- package/dist/managed-files/ci-workflow.d.ts +4 -1
- package/dist/managed-files/ci-workflow.d.ts.map +1 -1
- package/dist/managed-files/ci-workflow.js +9 -14
- package/dist/managed-files/files.d.ts +3 -3
- package/dist/managed-files/files.js +3 -3
- package/dist/managed-files/generator.d.ts +5 -1
- package/dist/managed-files/generator.d.ts.map +1 -1
- package/dist/managed-files/generator.js +22 -7
- package/dist/managed-files/render-memo.d.ts +13 -0
- package/dist/managed-files/render-memo.d.ts.map +1 -0
- package/dist/managed-files/render-memo.js +94 -0
- package/dist/task-results.js +2 -1
- package/dist/test-program.d.ts +11 -6
- package/dist/test-program.d.ts.map +1 -1
- package/dist/test-program.js +39 -35
- package/dist/tool-bins.d.ts +12 -0
- package/dist/tool-bins.d.ts.map +1 -1
- package/dist/tool-bins.js +31 -6
- package/dist/typecheck-test-policy.d.ts +10 -1
- package/dist/typecheck-test-policy.d.ts.map +1 -1
- package/dist/typecheck-test-policy.js +40 -24
- package/dist/workspace-config-policy.d.ts.map +1 -1
- package/dist/workspace-config-policy.js +135 -42
- package/dist/workspace-index.d.ts +14 -1
- package/dist/workspace-index.d.ts.map +1 -1
- package/dist/workspace-index.js +36 -4
- package/executors.json +2 -0
- package/managed/raw/envrc +24 -0
- package/managed/raw/patches/nx@24.0.3.patch +1851 -221
- package/managed/raw/patches/ttsc@0.37.1.patch +1055 -0
- package/managed/raw/tooling/direnv/developer-links.ts +122 -14
- package/managed/raw/tooling/direnv/devenv.smoo.nix +20 -19
- package/managed/templates/github/actions/save-nix-devenv/action.yml +19 -1
- package/managed/templates/github/actions/setup-devenv/action.yml +2 -0
- package/package.json +68 -11
- package/runtime/nx-publish/.build-inputs.json +2 -2
- package/runtime/nx-publish/cache_adapter.js +293 -3
- package/runtime/nx-publish/nx_publish.js +36 -14
- package/src/built-output-packages.ts +1 -1
- package/src/bun-lock-scope.ts +293 -40
- package/src/cargo-closure-input.ts +20 -12
- package/src/cargo-source-hash.ts +119 -145
- package/src/cargo-toolchain-paths.ts +2 -2
- package/src/cargo-workspace.ts +59 -21
- package/src/ensure-built-receipt.ts +3 -2
- package/src/eslint-config-closure.ts +27 -6
- package/src/executors/bounded-exec/admission-queue.ts +11 -10
- package/src/executors/bounded-exec/awake-clock.ts +20 -0
- package/src/executors/bounded-exec/bounds.ts +41 -19
- package/src/executors/bounded-exec/case-gate.ts +365 -0
- package/src/executors/bounded-exec/cpu-pool-thread.ts +3 -3
- package/src/executors/bounded-exec/cpu-pool.ts +139 -12
- package/src/executors/bounded-exec/cpu-rise.ts +156 -0
- package/src/executors/bounded-exec/endpoint-life.ts +279 -0
- package/src/executors/bounded-exec/executor.ts +341 -123
- package/src/executors/bounded-exec/gateway-lease.ts +6 -0
- package/src/executors/bounded-exec/go-run-summary.ts +137 -0
- package/src/executors/bounded-exec/hang-diagnostics.ts +150 -50
- package/src/executors/bounded-exec/lease-wait.ts +1 -0
- package/src/executors/bounded-exec/mbx-pool.ts +57 -9
- package/src/executors/bounded-exec/mbx-reservations.ts +216 -183
- package/src/executors/bounded-exec/parent-grant.ts +3 -0
- package/src/executors/bounded-exec/pool-admission.ts +59 -2
- package/src/executors/bounded-exec/progress-channel.ts +129 -0
- package/src/executors/bounded-exec/progress.ts +39 -3
- package/src/executors/bounded-exec/ram-temp.ts +137 -17
- package/src/executors/bounded-exec/reservation-connection.ts +221 -0
- package/src/executors/bounded-exec/run-cpu-pool.ts +2 -0
- package/src/executors/bounded-exec/runner-sizing.ts +13 -8
- package/src/executors/bounded-exec/schema.json +1 -1
- package/src/executors/bounded-exec/schema.ts +21 -2
- package/src/executors/bounded-exec/suite-minimum.ts +81 -0
- package/src/executors/bounded-exec/test-result-writer.ts +3 -0
- package/src/executors/bounded-exec/test-results.ts +5 -0
- package/src/executors/bounded-exec/verdict.ts +83 -22
- package/src/executors/cargo-zigbuild/executor.ts +30 -19
- package/src/index.ts +357 -166
- package/src/managed-files/ci-workflow.ts +13 -17
- package/src/managed-files/files.ts +3 -3
- package/src/managed-files/generator.ts +21 -6
- package/src/managed-files/render-memo.ts +135 -0
- package/src/test-program.ts +36 -41
- package/src/tool-bins.ts +35 -6
- package/src/typecheck-test-policy.ts +49 -24
- package/src/workspace-config-policy.ts +161 -43
- package/src/workspace-index.ts +40 -5
- package/test-timings.json +157 -141
- package/managed/raw/patches/ttsc@0.36.3.patch +0 -1660
|
@@ -61,6 +61,7 @@ export function holdLease(
|
|
|
61
61
|
what: string,
|
|
62
62
|
bounds: LeaseBounds | CpuLeaseWait,
|
|
63
63
|
onQueued?: () => void,
|
|
64
|
+
withdrawn?: AbortSignal,
|
|
64
65
|
): Promise<HeldLease> {
|
|
65
66
|
// `Promise.withResolvers` would read better but needs lib es2024; this package inherits lib
|
|
66
67
|
// es2022 from tsconfig.base.json.
|
|
@@ -231,5 +232,10 @@ export function holdLease(
|
|
|
231
232
|
socketClosed = true;
|
|
232
233
|
unleased(`the lease connection for ${what} at ${socket} closed`);
|
|
233
234
|
});
|
|
235
|
+
if (withdrawn !== undefined) {
|
|
236
|
+
const withdraw = (): void => unleased(`the request for ${what} at ${socket} was withdrawn`);
|
|
237
|
+
if (withdrawn.aborted) withdraw();
|
|
238
|
+
else withdrawn.addEventListener('abort', withdraw, { once: true });
|
|
239
|
+
}
|
|
234
240
|
return promise;
|
|
235
241
|
}
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
import { spawnSync } from 'node:child_process';
|
|
2
|
+
import { readFileSync } from 'node:fs';
|
|
3
|
+
import { join, relative, sep } from 'node:path';
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* A Go run by its top-level tests. `go test -json` reports every event and sums none, and a consumer that keeps only
|
|
7
|
+
* part of a long output cannot count it, so the run that read every event states the sum where its output ends.
|
|
8
|
+
*/
|
|
9
|
+
export interface GoRunSummary {
|
|
10
|
+
readonly passed: number;
|
|
11
|
+
readonly failed: number;
|
|
12
|
+
readonly skipped: number;
|
|
13
|
+
readonly packages: number;
|
|
14
|
+
readonly failedTests: readonly GoFailedTest[];
|
|
15
|
+
/** Packages that failed with no failed test of their own: a build that did not compile, a `TestMain` that exited. */
|
|
16
|
+
readonly failedPackages: readonly string[];
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export interface GoFailedTest {
|
|
20
|
+
readonly package: string;
|
|
21
|
+
readonly test: string;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/** One package as `go list -f GO_LIST_FORMAT` prints it: its directory and its in-package and external test files. */
|
|
25
|
+
export interface GoPackageFiles {
|
|
26
|
+
readonly package: string;
|
|
27
|
+
readonly directory: string;
|
|
28
|
+
readonly files: readonly string[];
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export const GO_LIST_FORMAT = '{{.ImportPath}}\t{{.Dir}}\t{{join .TestGoFiles ","}}\t{{join .XTestGoFiles ","}}';
|
|
32
|
+
|
|
33
|
+
export function parseGoList(stdout: string): GoPackageFiles[] {
|
|
34
|
+
const packages: GoPackageFiles[] = [];
|
|
35
|
+
for (const line of stdout.split('\n')) {
|
|
36
|
+
const [name, directory, own, external] = line.split('\t');
|
|
37
|
+
if (name === undefined || name === '' || directory === undefined || directory === '') continue;
|
|
38
|
+
packages.push({
|
|
39
|
+
package: name,
|
|
40
|
+
directory,
|
|
41
|
+
files: [own ?? '', external ?? ''].flatMap((list) => (list === '' ? [] : list.split(','))),
|
|
42
|
+
});
|
|
43
|
+
}
|
|
44
|
+
return packages;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* The file of a package that declares the top-level function `test`, and its line. Go admits one such function per
|
|
49
|
+
* test package, and a directory holds two test packages (in-package and external): a name both declare has no one
|
|
50
|
+
* file, and neither does a name none declares.
|
|
51
|
+
*/
|
|
52
|
+
export function declarationOf(
|
|
53
|
+
test: string,
|
|
54
|
+
files: readonly { readonly path: string; readonly text: string }[],
|
|
55
|
+
): { path: string; line: number } | null {
|
|
56
|
+
const declares = new RegExp(`^func\\s+${test.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\s*\\(`);
|
|
57
|
+
let found: { path: string; line: number } | null = null;
|
|
58
|
+
for (const file of files) {
|
|
59
|
+
const line = file.text.split('\n').findIndex((text) => declares.test(text));
|
|
60
|
+
if (line === -1) continue;
|
|
61
|
+
if (found !== null) return null;
|
|
62
|
+
found = { path: file.path, line: line + 1 };
|
|
63
|
+
}
|
|
64
|
+
return found;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/** Each failed test whose declaration is found, as `<workspace path>:<line>`, keyed `<package>::<test>`. */
|
|
68
|
+
export function goDeclarations(
|
|
69
|
+
failed: readonly GoFailedTest[],
|
|
70
|
+
root: string,
|
|
71
|
+
listed: readonly GoPackageFiles[],
|
|
72
|
+
read: (path: string) => string | null,
|
|
73
|
+
): Map<string, string> {
|
|
74
|
+
const declared = new Map<string, string>();
|
|
75
|
+
for (const { package: name, test } of failed) {
|
|
76
|
+
const found = listed.find((entry) => entry.package === name);
|
|
77
|
+
if (found === undefined) continue;
|
|
78
|
+
const files = found.files.flatMap((file) => {
|
|
79
|
+
const path = join(found.directory, file);
|
|
80
|
+
const text = read(path);
|
|
81
|
+
return text === null ? [] : [{ path, text }];
|
|
82
|
+
});
|
|
83
|
+
const declaration = declarationOf(test, files);
|
|
84
|
+
if (declaration === null) continue;
|
|
85
|
+
declared.set(`${name}::${test}`, `${relative(root, declaration.path).split(sep).join('/')}:${declaration.line}`);
|
|
86
|
+
}
|
|
87
|
+
return declared;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* The lines a Go run's output ends with. A failed test is named at its declaration, the identity a reader matches a
|
|
92
|
+
* source file by; one whose declaration was not found keeps its package, which no reader takes for a file.
|
|
93
|
+
*/
|
|
94
|
+
export function goRunSummaryText(summary: GoRunSummary, declared: ReadonlyMap<string, string>): string {
|
|
95
|
+
const lines = [
|
|
96
|
+
`go test summary: ${summary.passed} passed, ${summary.failed} failed, ${summary.skipped} skipped, ${summary.packages} packages`,
|
|
97
|
+
...summary.failedTests.map(
|
|
98
|
+
({ package: name, test }) => `go test failed: ${declared.get(`${name}::${test}`) ?? name} ${test}`,
|
|
99
|
+
),
|
|
100
|
+
...summary.failedPackages.map((name) => `go test package failed: ${name}`),
|
|
101
|
+
];
|
|
102
|
+
return `${lines.join('\n')}\n`;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/** How long `go list` may take to name the failed packages' directories and test files. */
|
|
106
|
+
const GO_LIST_MS = 60_000;
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* The declarations of a run's failed tests, asked of the toolchain that ran them: `go list` names each failed package's
|
|
110
|
+
* directory and test files, and the file that declares the test is found among them. A list that does not answer
|
|
111
|
+
* leaves every test at its package, which the summary then prints.
|
|
112
|
+
*/
|
|
113
|
+
export function declaredGoTests(
|
|
114
|
+
failed: readonly GoFailedTest[],
|
|
115
|
+
cwd: string,
|
|
116
|
+
root: string,
|
|
117
|
+
env: NodeJS.ProcessEnv,
|
|
118
|
+
): Map<string, string> {
|
|
119
|
+
if (failed.length === 0) return new Map();
|
|
120
|
+
const packages = [...new Set(failed.map((test) => test.package))];
|
|
121
|
+
const listed = spawnSync('go', ['list', '-f', GO_LIST_FORMAT, ...packages], {
|
|
122
|
+
cwd,
|
|
123
|
+
env,
|
|
124
|
+
encoding: 'utf8',
|
|
125
|
+
timeout: GO_LIST_MS,
|
|
126
|
+
});
|
|
127
|
+
if (listed.status !== 0) return new Map();
|
|
128
|
+
return goDeclarations(failed, root, parseGoList(listed.stdout), (path) => {
|
|
129
|
+
try {
|
|
130
|
+
return readFileSync(path, 'utf8');
|
|
131
|
+
} catch (error) {
|
|
132
|
+
// A listed file that cannot be read declares nothing here; any other failure is a bug to see.
|
|
133
|
+
if (error instanceof Error && 'code' in error) return null;
|
|
134
|
+
throw error;
|
|
135
|
+
}
|
|
136
|
+
});
|
|
137
|
+
}
|
|
@@ -1,6 +1,17 @@
|
|
|
1
1
|
import { spawn } from 'node:child_process';
|
|
2
2
|
import { describeTeardown, killProcessGroupNow, TEARDOWN_PATIENCE } from '../../process-group.js';
|
|
3
3
|
import { type ProcessEntry, withDescendants } from '../../testing.js';
|
|
4
|
+
import { atClockDeadline } from './awake-clock.js';
|
|
5
|
+
import {
|
|
6
|
+
CPU_RISING_CLASS,
|
|
7
|
+
type CpuRise,
|
|
8
|
+
type CpuTime,
|
|
9
|
+
cpuChange,
|
|
10
|
+
cpuRiseThresholdsMs,
|
|
11
|
+
describeCpuRise,
|
|
12
|
+
describeCpuThresholds,
|
|
13
|
+
parseCpuTime,
|
|
14
|
+
} from './cpu-rise.js';
|
|
4
15
|
import {
|
|
5
16
|
describeHolders,
|
|
6
17
|
type LockWait,
|
|
@@ -16,12 +27,20 @@ import {
|
|
|
16
27
|
*/
|
|
17
28
|
export const HANG_CAPTURE_MS = 10_000;
|
|
18
29
|
const MAX_PROBE_BYTES = 4 * 1024 * 1024;
|
|
19
|
-
const PS_FIELDS = 'pid=,ppid=,pgid=,stat=,%cpu=,etime=,lstart=,command=';
|
|
30
|
+
const PS_FIELDS = 'pid=,ppid=,pgid=,stat=,%cpu=,etime=,cputime=,lstart=,command=';
|
|
31
|
+
/**
|
|
32
|
+
* The least time between the two process censuses, which are the snapshot's two CPU samples. A process is named
|
|
33
|
+
* after gaining 2 s where ps counts whole seconds and 0.5 s where it counts centiseconds (cpuRiseThresholdMs), so five
|
|
34
|
+
* seconds show a process at 40% or at 10% of a core. On Darwin the stack samples have usually spent most of it;
|
|
35
|
+
* elsewhere, or when sudo refuses them, the snapshot pauses, inside its window and early enough that the closing
|
|
36
|
+
* census keeps its own limit.
|
|
37
|
+
*/
|
|
38
|
+
const CPU_INTERVAL_MS = 5_000;
|
|
20
39
|
/**
|
|
21
40
|
* Stack samples: `sample` suspends a process every millisecond for one second and reports its threads' call graph. The
|
|
22
|
-
* runner and at most this many leaf processes are sampled, all at once
|
|
23
|
-
*
|
|
24
|
-
* capture all the same.
|
|
41
|
+
* runner and at most this many leaf processes are sampled, all at once, and then at most this many named lock waiters
|
|
42
|
+
* those left out. `-mayDie` is not passed: it reads symbols before sampling and measured about a second slower under
|
|
43
|
+
* load, and a process that exits mid-sample is a raced capture all the same.
|
|
25
44
|
*/
|
|
26
45
|
const MAX_SAMPLED_LEAVES = 4;
|
|
27
46
|
const SAMPLE_SECONDS = 1;
|
|
@@ -58,14 +77,19 @@ export interface HangDiagnostics {
|
|
|
58
77
|
outcome: 'holders' | 'none' | 'incomplete';
|
|
59
78
|
/** Lock waits the snapshot names: empty unless the capture is complete, and always empty without the open-file census. */
|
|
60
79
|
lockWaits: readonly LockWait[];
|
|
80
|
+
/**
|
|
81
|
+
* The processes of the tree whose CPU time rose between the two censuses, largest gain first: empty when none did, and
|
|
82
|
+
* when the closing census did not complete. A build inside the test, never a restart of the idle bound.
|
|
83
|
+
*/
|
|
84
|
+
cpuRising: readonly CpuRise[];
|
|
61
85
|
text: string;
|
|
62
86
|
}
|
|
63
87
|
|
|
64
88
|
export interface HangCaptureRequest {
|
|
65
89
|
pid: number;
|
|
66
90
|
pgid: number | null;
|
|
67
|
-
|
|
68
|
-
|
|
91
|
+
awakeExpiresAtMs: number;
|
|
92
|
+
awakeNow(): number;
|
|
69
93
|
signal?: AbortSignal;
|
|
70
94
|
}
|
|
71
95
|
|
|
@@ -77,24 +101,44 @@ export type HangProbeResult =
|
|
|
77
101
|
export interface HangDiagnosticHost {
|
|
78
102
|
platform: NodeJS.Platform;
|
|
79
103
|
probe(file: string, args: readonly string[], timeoutMs: number, signal: AbortSignal): Promise<HangProbeResult>;
|
|
104
|
+
/** Waits `ms`, or until `signal` aborts: the capture's end ends its own CPU interval. */
|
|
105
|
+
sleep(ms: number, signal: AbortSignal): Promise<void>;
|
|
80
106
|
}
|
|
81
107
|
|
|
82
108
|
interface ObservedProcess extends ProcessEntry {
|
|
83
109
|
pgid: number;
|
|
84
110
|
started: string;
|
|
111
|
+
cpu: CpuTime | null;
|
|
85
112
|
raw: string;
|
|
86
113
|
}
|
|
87
114
|
|
|
88
|
-
|
|
115
|
+
function sleepUnlessAborted(ms: number, signal: AbortSignal, now: () => number): Promise<void> {
|
|
116
|
+
return new Promise((resolve) => {
|
|
117
|
+
const done = (): void => {
|
|
118
|
+
cancelTimer();
|
|
119
|
+
signal.removeEventListener('abort', done);
|
|
120
|
+
resolve();
|
|
121
|
+
};
|
|
122
|
+
const cancelTimer = atClockDeadline(now() + ms, now, done);
|
|
123
|
+
signal.addEventListener('abort', done, { once: true });
|
|
124
|
+
if (signal.aborted) done();
|
|
125
|
+
});
|
|
126
|
+
}
|
|
89
127
|
|
|
90
128
|
/** A pre-expiry observation, never an assertion that an open file is a contested lock. */
|
|
91
129
|
export function captureHangDiagnostics(
|
|
92
130
|
request: HangCaptureRequest,
|
|
93
|
-
source: HangDiagnosticHost =
|
|
131
|
+
source: HangDiagnosticHost = {
|
|
132
|
+
platform: process.platform,
|
|
133
|
+
probe: (file, args, timeoutMs, signal) => runProbe(file, args, timeoutMs, signal, undefined, request.awakeNow),
|
|
134
|
+
sleep: (ms, signal) => sleepUnlessAborted(ms, signal, request.awakeNow),
|
|
135
|
+
},
|
|
94
136
|
): Promise<HangDiagnostics> {
|
|
95
137
|
const startedAtMs = Date.now();
|
|
96
|
-
const
|
|
97
|
-
const
|
|
138
|
+
const { awakeNow, awakeExpiresAtMs } = request;
|
|
139
|
+
const startedAwake = awakeNow();
|
|
140
|
+
const endsAtMs = Math.min(startedAwake + HANG_CAPTURE_MS, awakeExpiresAtMs);
|
|
141
|
+
const budgetMs = Math.max(0, Math.min(HANG_CAPTURE_MS, awakeExpiresAtMs - startedAwake));
|
|
98
142
|
const controller = new AbortController();
|
|
99
143
|
const lines = [
|
|
100
144
|
`Hang snapshot: pid=${request.pid} pgid=${request.pgid ?? 'unknown'} startedAtMs=${startedAtMs}`,
|
|
@@ -104,29 +148,33 @@ export function captureHangDiagnostics(
|
|
|
104
148
|
const during = (): string => (active.size === 0 ? 'process census' : [...active].join(', '));
|
|
105
149
|
let finished = false;
|
|
106
150
|
let waits: readonly LockWait[] = [];
|
|
151
|
+
let rising: readonly CpuRise[] = [];
|
|
107
152
|
// This package targets lib es2022, which does not declare Promise.withResolvers.
|
|
108
153
|
return new Promise((resolve) => {
|
|
109
154
|
const finish = (outcome: HangDiagnostics['outcome']): void => {
|
|
110
155
|
if (finished) return;
|
|
111
156
|
finished = true;
|
|
112
|
-
|
|
157
|
+
cancelTimer();
|
|
113
158
|
request.signal?.removeEventListener('abort', cancelled);
|
|
114
159
|
controller.abort();
|
|
115
160
|
const capturedAtMs = Date.now();
|
|
116
|
-
const offsetFromExpiryMs =
|
|
161
|
+
const offsetFromExpiryMs = awakeExpiresAtMs - awakeNow();
|
|
117
162
|
resolve({
|
|
118
163
|
startedAtMs,
|
|
119
164
|
capturedAtMs,
|
|
120
165
|
offsetFromExpiryMs,
|
|
121
166
|
outcome,
|
|
122
167
|
lockWaits: outcome === 'incomplete' ? [] : waits,
|
|
168
|
+
cpuRising: rising,
|
|
123
169
|
text: `${lines.join('\n')}\nHang snapshot outcome=${outcome} capturedAtMs=${capturedAtMs} offsetFromExpiryMs=${offsetFromExpiryMs}\n`,
|
|
124
170
|
});
|
|
125
171
|
};
|
|
126
|
-
const
|
|
127
|
-
lines.push(
|
|
172
|
+
const cancelTimer = atClockDeadline(endsAtMs, awakeNow, () => {
|
|
173
|
+
lines.push(
|
|
174
|
+
`Capture incomplete: ${Math.ceil(budgetMs)}ms deadline reached during ${during()}; no empty-holder conclusion.`,
|
|
175
|
+
);
|
|
128
176
|
finish('incomplete');
|
|
129
|
-
}
|
|
177
|
+
});
|
|
130
178
|
const cancelled = (): void => {
|
|
131
179
|
if (finished) return;
|
|
132
180
|
lines.push(`Capture cancelled during ${during()}; its progress generation is no longer current.`);
|
|
@@ -148,7 +196,7 @@ export function captureHangDiagnostics(
|
|
|
148
196
|
const result = await source.probe(
|
|
149
197
|
file,
|
|
150
198
|
args,
|
|
151
|
-
Math.max(0, Math.min(limitMs, endsAtMs -
|
|
199
|
+
Math.max(0, Math.min(limitMs, endsAtMs - awakeNow())),
|
|
152
200
|
controller.signal,
|
|
153
201
|
);
|
|
154
202
|
active.delete(name);
|
|
@@ -171,31 +219,41 @@ export function captureHangDiagnostics(
|
|
|
171
219
|
const parsed: ObservedProcess[] = [];
|
|
172
220
|
for (const row of result.stdout.split('\n').filter((line) => line.trim() !== '')) {
|
|
173
221
|
const match =
|
|
174
|
-
/^\s*(\d+)\s+(\d+)\s+(\d+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+\s+\S+\s+\d+\s+\d\d:\d\d:\d\d\s+\d{4})\s+(.*)$/.exec(
|
|
222
|
+
/^\s*(\d+)\s+(\d+)\s+(\d+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+\s+\S+\s+\d+\s+\d\d:\d\d:\d\d\s+\d{4})\s+(.*)$/.exec(
|
|
175
223
|
row,
|
|
176
224
|
);
|
|
177
225
|
if (match === null) {
|
|
178
226
|
lines.push(`${name}: could not decode ps row: ${row}`);
|
|
179
227
|
return null;
|
|
180
228
|
}
|
|
181
|
-
const [, pid, ppid, pgid, stat, , , started, command] = match;
|
|
229
|
+
const [, pid, ppid, pgid, stat, , , cputime, started, command] = match;
|
|
182
230
|
if (
|
|
183
231
|
pid === undefined ||
|
|
184
232
|
ppid === undefined ||
|
|
185
233
|
pgid === undefined ||
|
|
186
234
|
stat === undefined ||
|
|
235
|
+
cputime === undefined ||
|
|
187
236
|
started === undefined ||
|
|
188
237
|
command === undefined
|
|
189
238
|
) {
|
|
190
239
|
lines.push(`${name}: incomplete ps row: ${row}`);
|
|
191
240
|
return null;
|
|
192
241
|
}
|
|
193
|
-
parsed.push({
|
|
242
|
+
parsed.push({
|
|
243
|
+
pid: Number(pid),
|
|
244
|
+
ppid: Number(ppid),
|
|
245
|
+
pgid: Number(pgid),
|
|
246
|
+
stat,
|
|
247
|
+
cpu: parseCpuTime(cputime),
|
|
248
|
+
started,
|
|
249
|
+
command,
|
|
250
|
+
raw: row,
|
|
251
|
+
});
|
|
194
252
|
}
|
|
195
253
|
const roots = [request.pid, ...parsed.filter((entry) => entry.pgid === request.pgid).map((entry) => entry.pid)];
|
|
196
254
|
const selected = new Set(withDescendants(parsed, roots).map((entry) => entry.pid));
|
|
197
255
|
const tree = parsed.filter((entry) => selected.has(entry.pid));
|
|
198
|
-
lines.push(`${name}: PID PPID PGID STAT %CPU ELAPSED STARTED COMMAND`);
|
|
256
|
+
lines.push(`${name}: PID PPID PGID STAT %CPU ELAPSED CPUTIME STARTED COMMAND`);
|
|
199
257
|
lines.push(...tree.map((entry) => entry.raw));
|
|
200
258
|
if (!tree.some((entry) => entry.pid === request.pid)) {
|
|
201
259
|
lines.push(`${name}: root pid ${request.pid} exited or is not visible; capture raced or is incomplete.`);
|
|
@@ -203,40 +261,45 @@ export function captureHangDiagnostics(
|
|
|
203
261
|
}
|
|
204
262
|
return tree;
|
|
205
263
|
};
|
|
206
|
-
/**
|
|
207
|
-
|
|
208
|
-
* process waits in, which the open-file census cannot. A sample sudo refuses, or one that does not answer, is named
|
|
209
|
-
* as no stack for that process; stacks never change the holder outcome.
|
|
210
|
-
*/
|
|
211
|
-
const stackSamples = async (tree: readonly ObservedProcess[]): Promise<void> => {
|
|
264
|
+
/** The runner and up to MAX_SAMPLED_LEAVES leaf processes of `tree` (members with no live child), and the leaves past that bound. */
|
|
265
|
+
const leafSamples = (tree: readonly ObservedProcess[]): { sampled: number[]; omitted: number[] } => {
|
|
212
266
|
const live = tree.filter((entry) => !entry.stat.startsWith('Z'));
|
|
213
267
|
const parents = new Set(live.map((entry) => entry.ppid));
|
|
214
268
|
const leaves = live.filter((entry) => entry.pid !== request.pid && !parents.has(entry.pid));
|
|
215
|
-
|
|
269
|
+
return {
|
|
270
|
+
sampled: [request.pid, ...leaves.slice(0, MAX_SAMPLED_LEAVES).map((entry) => entry.pid)],
|
|
271
|
+
omitted: leaves.slice(MAX_SAMPLED_LEAVES).map((entry) => entry.pid),
|
|
272
|
+
};
|
|
273
|
+
};
|
|
274
|
+
/**
|
|
275
|
+
* One one-second call stack of `pid`, until the closing census would have less than its own limit left: a stack
|
|
276
|
+
* names the call a process waits in, which the open-file census cannot. A sample sudo refuses, or one that does
|
|
277
|
+
* not answer, is named as no stack for that process; stacks never change the holder outcome.
|
|
278
|
+
*/
|
|
279
|
+
const sampleStack = async (pid: number): Promise<void> => {
|
|
280
|
+
const name = `stack sample pid ${pid}`;
|
|
281
|
+
// `-file /dev/stdout` writes the report to the probe's pipe alone, not also to a root-owned file in /tmp.
|
|
282
|
+
const args = ['-n', 'sample', String(pid), String(SAMPLE_SECONDS), '-file', '/dev/stdout'];
|
|
283
|
+
const result = await probe(name, 'sudo', args, endsAtMs - awakeNow() - CENSUS_LIMIT_MS);
|
|
284
|
+
if (finished) return;
|
|
285
|
+
const report = callGraph(result.stdout);
|
|
286
|
+
if (result.status === 'exited' && result.code === 0 && report !== '') {
|
|
287
|
+
lines.push(`${name} (${SAMPLE_SECONDS}s; binary image list omitted):\n${report}`);
|
|
288
|
+
return;
|
|
289
|
+
}
|
|
290
|
+
if (report !== '') lines.push(`Partial ${name} output:\n${report}`);
|
|
291
|
+
lines.push(
|
|
292
|
+
`${name}: no stack; sudo -n sample did not answer, so the call this process waits in is not observed.`,
|
|
293
|
+
);
|
|
294
|
+
};
|
|
295
|
+
/** The runner's and its tree's leaf processes' stacks, taken at once. */
|
|
296
|
+
const stackSamples = async (tree: readonly ObservedProcess[]): Promise<void> => {
|
|
297
|
+
const { sampled, omitted } = leafSamples(tree);
|
|
216
298
|
if (omitted.length > 0)
|
|
217
299
|
lines.push(
|
|
218
300
|
`Stack samples bounded to the runner and ${MAX_SAMPLED_LEAVES} leaf processes; not sampled: pid ${omitted.join(', ')}`,
|
|
219
301
|
);
|
|
220
|
-
|
|
221
|
-
const sampled = [request.pid, ...leaves.slice(0, MAX_SAMPLED_LEAVES).map((entry) => entry.pid)];
|
|
222
|
-
await Promise.all(
|
|
223
|
-
sampled.map(async (pid) => {
|
|
224
|
-
const name = `stack sample pid ${pid}`;
|
|
225
|
-
// `-file /dev/stdout` writes the report to the probe's pipe alone, not also to a root-owned file in /tmp.
|
|
226
|
-
const args = ['-n', 'sample', String(pid), String(SAMPLE_SECONDS), '-file', '/dev/stdout'];
|
|
227
|
-
const result = await probe(name, 'sudo', args, limitMs);
|
|
228
|
-
if (finished) return;
|
|
229
|
-
const report = callGraph(result.stdout);
|
|
230
|
-
if (result.status === 'exited' && result.code === 0 && report !== '') {
|
|
231
|
-
lines.push(`${name} (${SAMPLE_SECONDS}s; binary image list omitted):\n${report}`);
|
|
232
|
-
return;
|
|
233
|
-
}
|
|
234
|
-
if (report !== '') lines.push(`Partial ${name} output:\n${report}`);
|
|
235
|
-
lines.push(
|
|
236
|
-
`${name}: no stack; sudo -n sample did not answer, so the call this process waits in is not observed.`,
|
|
237
|
-
);
|
|
238
|
-
}),
|
|
239
|
-
);
|
|
302
|
+
await Promise.all(sampled.map(sampleStack));
|
|
240
303
|
};
|
|
241
304
|
/**
|
|
242
305
|
* A process in an uninterruptible kernel wait (state U) is in a file-system or device call no signal ends, and its
|
|
@@ -246,7 +309,7 @@ export function captureHangDiagnostics(
|
|
|
246
309
|
*/
|
|
247
310
|
const kernelWaits = async (tree: readonly ObservedProcess[]): Promise<void> => {
|
|
248
311
|
const waiting = tree.filter((entry) => entry.stat.startsWith('U')).slice(0, MAX_SAMPLED_LEAVES);
|
|
249
|
-
const limitMs = endsAtMs -
|
|
312
|
+
const limitMs = endsAtMs - awakeNow() - CENSUS_LIMIT_MS;
|
|
250
313
|
await Promise.all(
|
|
251
314
|
waiting.map(async ({ pid }) => {
|
|
252
315
|
const name = `fs_usage pid ${pid}`;
|
|
@@ -267,6 +330,7 @@ export function captureHangDiagnostics(
|
|
|
267
330
|
const before = await census('ps before');
|
|
268
331
|
if (finished) return;
|
|
269
332
|
if (before === null) return finish('incomplete');
|
|
333
|
+
const beforeAtMs = awakeNow();
|
|
270
334
|
let incomplete = false;
|
|
271
335
|
let observed = before.length > 0;
|
|
272
336
|
if (source.platform === 'darwin') {
|
|
@@ -318,6 +382,11 @@ export function captureHangDiagnostics(
|
|
|
318
382
|
lines.push('Lock holder census incomplete; no lock wait is named.');
|
|
319
383
|
} else {
|
|
320
384
|
waits = lockWaits(rows, new Set(before.map((entry) => entry.pid)), openers);
|
|
385
|
+
// A lock wait is open-handle evidence: only the waiter's own stack says whether it blocks in the lock
|
|
386
|
+
// call. Each named waiter the leaf samples left out is sampled the same way, inside the same window.
|
|
387
|
+
const stacked = new Set(leafSamples(before).sampled);
|
|
388
|
+
const waiters = [...new Set(waits.flatMap((wait) => wait.waiters))].filter((pid) => !stacked.has(pid));
|
|
389
|
+
await Promise.all(waiters.slice(0, MAX_SAMPLED_LEAVES).map(sampleStack));
|
|
321
390
|
}
|
|
322
391
|
}
|
|
323
392
|
};
|
|
@@ -327,9 +396,17 @@ export function captureHangDiagnostics(
|
|
|
327
396
|
lines.push(`Open-file census not provided on ${source.platform}; observations below name processes only.`);
|
|
328
397
|
lines.push(`Stack samples not provided on ${source.platform}; no blocking call is observed.`);
|
|
329
398
|
}
|
|
399
|
+
const settleMs = Math.min(beforeAtMs + CPU_INTERVAL_MS, endsAtMs - CENSUS_LIMIT_MS) - awakeNow();
|
|
400
|
+
if (settleMs > 0) {
|
|
401
|
+
active.add('CPU interval');
|
|
402
|
+
await source.sleep(settleMs, controller.signal);
|
|
403
|
+
active.delete('CPU interval');
|
|
404
|
+
if (finished) return;
|
|
405
|
+
}
|
|
330
406
|
const after = await census('ps after');
|
|
331
407
|
if (finished) return;
|
|
332
408
|
if (after === null) return finish('incomplete');
|
|
409
|
+
const afterAtMs = awakeNow();
|
|
333
410
|
if (
|
|
334
411
|
before.length !== after.length ||
|
|
335
412
|
before.some(
|
|
@@ -348,6 +425,28 @@ export function captureHangDiagnostics(
|
|
|
348
425
|
incomplete = true;
|
|
349
426
|
lines.push('Process identities or membership changed during capture; open-file census raced.');
|
|
350
427
|
}
|
|
428
|
+
const change = cpuChange(before, after);
|
|
429
|
+
const thresholdsMs = cpuRiseThresholdsMs(before, after);
|
|
430
|
+
rising = change.rising;
|
|
431
|
+
const intervalMs = afterAtMs - beforeAtMs;
|
|
432
|
+
if (change.unread.length > 0) {
|
|
433
|
+
lines.push(
|
|
434
|
+
`CPU time not read for pid ${change.unread.join(', ')}: ps printed none that parses, so they are not counted idle.`,
|
|
435
|
+
);
|
|
436
|
+
}
|
|
437
|
+
if (rising.length > 0) {
|
|
438
|
+
lines.push(
|
|
439
|
+
`CPU rose in the tree over about ${intervalMs}ms between the censuses (a rise is named from ${describeCpuThresholds(thresholdsMs)} of CPU):`,
|
|
440
|
+
...rising.map((rise) => ` ${describeCpuRise(rise)}`),
|
|
441
|
+
`Classification: ${CPU_RISING_CLASS}`,
|
|
442
|
+
);
|
|
443
|
+
} else if (thresholdsMs.length === 0) {
|
|
444
|
+
lines.push('CPU: no process of the tree had a CPU time ps gave in both censuses, so none was compared.');
|
|
445
|
+
} else {
|
|
446
|
+
lines.push(
|
|
447
|
+
`CPU: no process of the tree gained ${describeCpuThresholds(thresholdsMs)} of CPU over about ${intervalMs}ms between the censuses.`,
|
|
448
|
+
);
|
|
449
|
+
}
|
|
351
450
|
if (!incomplete) {
|
|
352
451
|
for (const wait of waits) {
|
|
353
452
|
lines.push(
|
|
@@ -389,6 +488,7 @@ export function runProbe(
|
|
|
389
488
|
timeoutMs: number,
|
|
390
489
|
signal: AbortSignal,
|
|
391
490
|
environment?: NodeJS.ProcessEnv,
|
|
491
|
+
now: () => number = () => performance.now(),
|
|
392
492
|
): Promise<HangProbeResult> {
|
|
393
493
|
// Keep the package's es2022-compatible event-completion convention.
|
|
394
494
|
return new Promise((resolve) => {
|
|
@@ -404,7 +504,7 @@ export function runProbe(
|
|
|
404
504
|
const finish = (result: HangProbeResult): void => {
|
|
405
505
|
if (settled) return;
|
|
406
506
|
settled = true;
|
|
407
|
-
|
|
507
|
+
cancelTimer();
|
|
408
508
|
signal.removeEventListener('abort', aborted);
|
|
409
509
|
child.stdout.destroy();
|
|
410
510
|
child.stderr.destroy();
|
|
@@ -426,7 +526,7 @@ export function runProbe(
|
|
|
426
526
|
finish({ status, detail: detail + killFailure, stdout, stderr });
|
|
427
527
|
};
|
|
428
528
|
const aborted = (): void => stop('capture deadline reached');
|
|
429
|
-
const
|
|
529
|
+
const cancelTimer = atClockDeadline(now() + timeoutMs, now, () => stop(`probe exceeded ${timeoutMs}ms`));
|
|
430
530
|
signal.addEventListener('abort', aborted, { once: true });
|
|
431
531
|
const append = (stream: 'stdout' | 'stderr', chunk: string): void => {
|
|
432
532
|
bytes += Buffer.byteLength(chunk);
|