@smoothbricks/nx-plugin 0.7.17 → 0.7.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +204 -64
- package/dist/.build-inputs.json +24 -21
- package/dist/bun-lock-scope.d.ts +128 -20
- package/dist/bun-lock-scope.d.ts.map +1 -1
- package/dist/bun-lock-scope.js +267 -44
- package/dist/cargo-closure-input.d.ts +8 -6
- package/dist/cargo-closure-input.d.ts.map +1 -1
- package/dist/cargo-closure-input.js +14 -11
- package/dist/cargo-source-hash.d.ts +23 -4
- package/dist/cargo-source-hash.d.ts.map +1 -1
- package/dist/cargo-source-hash.js +80 -14
- package/dist/cargo-workspace.d.ts +20 -2
- package/dist/cargo-workspace.d.ts.map +1 -1
- package/dist/cargo-workspace.js +42 -22
- package/dist/ensure-built-receipt.d.ts +3 -2
- package/dist/ensure-built-receipt.d.ts.map +1 -1
- package/dist/ensure-built-receipt.js +3 -2
- package/dist/eslint-config-closure.d.ts +8 -3
- package/dist/eslint-config-closure.d.ts.map +1 -1
- package/dist/eslint-config-closure.js +24 -5
- package/dist/executors/bounded-exec/bounds.d.ts +12 -1
- package/dist/executors/bounded-exec/bounds.d.ts.map +1 -1
- package/dist/executors/bounded-exec/bounds.js +8 -4
- package/dist/executors/bounded-exec/cpu-rise.d.ts +73 -0
- package/dist/executors/bounded-exec/cpu-rise.d.ts.map +1 -0
- package/dist/executors/bounded-exec/cpu-rise.js +111 -0
- package/dist/executors/bounded-exec/executor.d.ts +1 -1
- package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
- package/dist/executors/bounded-exec/executor.js +54 -22
- package/dist/executors/bounded-exec/hang-diagnostics.d.ts +8 -0
- package/dist/executors/bounded-exec/hang-diagnostics.d.ts.map +1 -1
- package/dist/executors/bounded-exec/hang-diagnostics.js +66 -6
- package/dist/executors/bounded-exec/mbx-pool.d.ts +16 -2
- package/dist/executors/bounded-exec/mbx-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/mbx-pool.js +18 -2
- package/dist/executors/bounded-exec/mbx-reservations.d.ts +5 -2
- package/dist/executors/bounded-exec/mbx-reservations.d.ts.map +1 -1
- package/dist/executors/bounded-exec/mbx-reservations.js +5 -3
- package/dist/executors/bounded-exec/progress-channel.d.ts +38 -0
- package/dist/executors/bounded-exec/progress-channel.d.ts.map +1 -0
- package/dist/executors/bounded-exec/progress-channel.js +113 -0
- package/dist/executors/bounded-exec/progress.d.ts +2 -1
- package/dist/executors/bounded-exec/progress.d.ts.map +1 -1
- package/dist/executors/bounded-exec/ram-temp.d.ts +24 -1
- package/dist/executors/bounded-exec/ram-temp.d.ts.map +1 -1
- package/dist/executors/bounded-exec/ram-temp.js +123 -16
- package/dist/executors/bounded-exec/schema.d.ts +5 -0
- package/dist/executors/bounded-exec/schema.d.ts.map +1 -1
- package/dist/executors/bounded-exec/schema.js +17 -1
- package/dist/executors/bounded-exec/verdict.d.ts +13 -6
- package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
- package/dist/executors/bounded-exec/verdict.js +34 -9
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +199 -89
- package/dist/managed-files/generator.d.ts +5 -1
- package/dist/managed-files/generator.d.ts.map +1 -1
- package/dist/managed-files/generator.js +22 -7
- package/dist/managed-files/render-memo.d.ts +13 -0
- package/dist/managed-files/render-memo.d.ts.map +1 -0
- package/dist/managed-files/render-memo.js +94 -0
- package/dist/test-program.d.ts +11 -6
- package/dist/test-program.d.ts.map +1 -1
- package/dist/test-program.js +39 -35
- package/dist/tool-bins.d.ts +12 -0
- package/dist/tool-bins.d.ts.map +1 -1
- package/dist/tool-bins.js +31 -6
- package/dist/typecheck-test-policy.d.ts +10 -1
- package/dist/typecheck-test-policy.d.ts.map +1 -1
- package/dist/typecheck-test-policy.js +40 -24
- package/dist/workspace-index.d.ts +14 -1
- package/dist/workspace-index.d.ts.map +1 -1
- package/dist/workspace-index.js +36 -4
- package/managed/raw/patches/nx@24.0.3.patch +703 -119
- package/managed/raw/tooling/direnv/developer-links.ts +17 -3
- package/managed/templates/github/actions/save-nix-devenv/action.yml +19 -1
- package/managed/templates/github/actions/setup-devenv/action.yml +2 -0
- package/package.json +3 -2
- package/src/bun-lock-scope.ts +293 -40
- package/src/cargo-closure-input.ts +19 -11
- package/src/cargo-source-hash.ts +103 -13
- package/src/cargo-workspace.ts +59 -20
- package/src/ensure-built-receipt.ts +3 -2
- package/src/eslint-config-closure.ts +27 -6
- package/src/executors/bounded-exec/bounds.ts +13 -4
- package/src/executors/bounded-exec/cpu-rise.ts +156 -0
- package/src/executors/bounded-exec/executor.ts +65 -21
- package/src/executors/bounded-exec/hang-diagnostics.ts +88 -6
- package/src/executors/bounded-exec/mbx-pool.ts +24 -3
- package/src/executors/bounded-exec/mbx-reservations.ts +7 -5
- package/src/executors/bounded-exec/progress-channel.ts +129 -0
- package/src/executors/bounded-exec/progress.ts +2 -1
- package/src/executors/bounded-exec/ram-temp.ts +137 -17
- package/src/executors/bounded-exec/schema.ts +19 -0
- package/src/executors/bounded-exec/verdict.ts +44 -16
- package/src/index.ts +255 -98
- package/src/managed-files/generator.ts +21 -6
- package/src/managed-files/render-memo.ts +135 -0
- package/src/test-program.ts +36 -41
- package/src/tool-bins.ts +35 -6
- package/src/typecheck-test-policy.ts +49 -24
- package/src/workspace-index.ts +40 -5
- package/test-timings.json +6 -0
|
@@ -7,7 +7,9 @@
|
|
|
7
7
|
* ceiling starts at spawn. Later zero-hold intervals pause only the idle budget.
|
|
8
8
|
* - Any explicitly queued task process pauses idle, including alongside held grants; ceilings keep running.
|
|
9
9
|
* - The fixture ceiling ends with its segment. The idle clock also stops outside the testing lifecycle.
|
|
10
|
-
* - Finished tests reset the idle budget whether their release byte arrived before or after their output
|
|
10
|
+
* - Finished tests reset the idle budget whether their release byte arrived before or after their output, and so does
|
|
11
|
+
* progress a running test reports on the run's progress channel (progress-channel.ts). The run's CPU is no event:
|
|
12
|
+
* it never resets the budget.
|
|
11
13
|
*/
|
|
12
14
|
export const UNARMED = {
|
|
13
15
|
idle: null,
|
|
@@ -94,12 +96,14 @@ export function stepBounds(bounds, limits, event) {
|
|
|
94
96
|
idleRemaining: bounds.idle === null ? bounds.idleRemaining : Math.max(0, bounds.idle - event.at),
|
|
95
97
|
};
|
|
96
98
|
}
|
|
97
|
-
case 'progress':
|
|
99
|
+
case 'progress':
|
|
100
|
+
case 'reported': {
|
|
101
|
+
const moved = event.kind === 'reported' || event.finished;
|
|
98
102
|
if (bounds.queued)
|
|
99
|
-
return
|
|
103
|
+
return moved ? { ...bounds, idleRemaining: limits.idleMs ?? null } : bounds;
|
|
100
104
|
if (!event.testing)
|
|
101
105
|
return { ...bounds, idle: null, idleRemaining: limits.idleMs ?? null };
|
|
102
|
-
const restart = event.
|
|
106
|
+
const restart = moved || (event.kind === 'progress' && !event.wasTesting);
|
|
103
107
|
if (!restart || limits.idleMs === undefined)
|
|
104
108
|
return bounds;
|
|
105
109
|
return {
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
import type { tags } from 'typia';
|
|
2
|
+
/**
|
|
3
|
+
* The CPU a run's tree used while its idle bound counted down. A test that runs a compiler is a build inside the test,
|
|
4
|
+
* which is a defect and not a test (specs/cowshed/08_testing.md, "Integration tests (real substrate)"): the pre-signal
|
|
5
|
+
* snapshot names the processes whose CPU time rose between its two process censuses, so the expiry fails as that and
|
|
6
|
+
* not as an anonymous hang. CPU never restarts the idle bound, which counts only finished tests and the progress a
|
|
7
|
+
* running test reports (progress-channel.ts); this says what ran.
|
|
8
|
+
*/
|
|
9
|
+
/** What a rising process makes of the expiry, said in the same words by the snapshot and by the verdict. */
|
|
10
|
+
export declare const CPU_RISING_CLASS = "a process in the test's tree was using CPU: likely a build inside the test";
|
|
11
|
+
/**
|
|
12
|
+
* The least gain that names a rise, whatever resolution ps prints CPU time at: 10% of a core over the snapshot's five
|
|
13
|
+
* seconds between censuses.
|
|
14
|
+
*/
|
|
15
|
+
export declare const CPU_RISE_FLOOR_MS = 500;
|
|
16
|
+
/**
|
|
17
|
+
* What a printed CPU time resolves to, in milliseconds: the value of its last digit. Whole seconds are 1,000, a
|
|
18
|
+
* fraction of n digits is 10^(3-n); a finer fraction is kept to the millisecond, so none is below 1.
|
|
19
|
+
*/
|
|
20
|
+
export type CpuResolutionMs = 1_000 | 100 | 10 | 1;
|
|
21
|
+
/** A CPU time ps printed, with the resolution it was printed at. */
|
|
22
|
+
export interface CpuTime {
|
|
23
|
+
readonly ms: number;
|
|
24
|
+
readonly resolutionMs: CpuResolutionMs;
|
|
25
|
+
}
|
|
26
|
+
/** A process whose CPU time rose between the snapshot's two censuses. */
|
|
27
|
+
export interface CpuRise {
|
|
28
|
+
readonly pid: number & tags.Type<'uint32'> & tags.Minimum<1>;
|
|
29
|
+
readonly command: string;
|
|
30
|
+
readonly gainedMs: number & tags.Minimum<0>;
|
|
31
|
+
}
|
|
32
|
+
/** One process of a census, as far as CPU is concerned. */
|
|
33
|
+
export interface CpuSample {
|
|
34
|
+
readonly pid: number;
|
|
35
|
+
readonly started: string;
|
|
36
|
+
readonly command: string;
|
|
37
|
+
/** Null when ps printed a CPU time that does not parse. */
|
|
38
|
+
readonly cpu: CpuTime | null;
|
|
39
|
+
}
|
|
40
|
+
export interface CpuChange {
|
|
41
|
+
/** Largest gain first. */
|
|
42
|
+
readonly rising: readonly CpuRise[];
|
|
43
|
+
/** Processes in both censuses that one of them gave no readable CPU time for: not counted idle. */
|
|
44
|
+
readonly unread: readonly number[];
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* A ps CPU time (`cputime=`): `[[D-]HH:]MM:SS` with an optional fraction, as Linux prints it (whole seconds) and macOS
|
|
48
|
+
* does (centiseconds). Null for anything else, such as the `-` ps prints for a process it may not read.
|
|
49
|
+
*/
|
|
50
|
+
export declare function parseCpuTime(text: string): CpuTime | null;
|
|
51
|
+
/**
|
|
52
|
+
* The least gain between two readings of one process that names a rise: twice the coarser of their resolutions, and
|
|
53
|
+
* never under {@link CPU_RISE_FLOOR_MS}. A process that crossed a boundary of the last printed digit on a few
|
|
54
|
+
* milliseconds of work gains one resolution, so two are more than one. That is 2,000 ms between whole-second readings
|
|
55
|
+
* and 500 ms between centisecond ones.
|
|
56
|
+
*/
|
|
57
|
+
export declare function cpuRiseThresholdMs(before: CpuTime, after: CpuTime): number;
|
|
58
|
+
/**
|
|
59
|
+
* The processes present in both censuses (same pid and start) whose CPU time rose by at least
|
|
60
|
+
* {@link cpuRiseThresholdMs}.
|
|
61
|
+
*/
|
|
62
|
+
export declare function cpuChange(before: readonly CpuSample[], after: readonly CpuSample[]): CpuChange;
|
|
63
|
+
/**
|
|
64
|
+
* The gains {@link cpuChange} held these censuses to, each distinct one once, smallest first: one per resolution of the
|
|
65
|
+
* readings compared, and none when no process had a readable CPU time in both.
|
|
66
|
+
*/
|
|
67
|
+
export declare function cpuRiseThresholdsMs(before: readonly CpuSample[], after: readonly CpuSample[]): readonly number[];
|
|
68
|
+
/** The gains of {@link cpuRiseThresholdsMs} in seconds, as the snapshot says them: `2 s`, or `0.5 s or 2 s`. */
|
|
69
|
+
export declare function describeCpuThresholds(thresholdsMs: readonly number[]): string;
|
|
70
|
+
export declare function describeCpuRise(rise: CpuRise): string;
|
|
71
|
+
/** The largest gains, named, and how many more there were: a parallel build runs many processes. */
|
|
72
|
+
export declare function describeCpuRises(rises: readonly CpuRise[], named?: number): string;
|
|
73
|
+
//# sourceMappingURL=cpu-rise.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cpu-rise.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/cpu-rise.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,OAAO,CAAC;AAElC;;;;;;GAMG;AAEH,4GAA4G;AAC5G,eAAO,MAAM,gBAAgB,+EAA+E,CAAC;AAE7G;;;GAGG;AACH,eAAO,MAAM,iBAAiB,MAAM,CAAC;AAErC;;;GAGG;AACH,MAAM,MAAM,eAAe,GAAG,KAAK,GAAG,GAAG,GAAG,EAAE,GAAG,CAAC,CAAC;AAEnD,oEAAoE;AACpE,MAAM,WAAW,OAAO;IACtB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,YAAY,EAAE,eAAe,CAAC;CACxC;AAKD,yEAAyE;AACzE,MAAM,WAAW,OAAO;IACtB,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAC7D,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;CAC7C;AAED,2DAA2D;AAC3D,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,2DAA2D;IAC3D,QAAQ,CAAC,GAAG,EAAE,OAAO,GAAG,IAAI,CAAC;CAC9B;AAED,MAAM,WAAW,SAAS;IACxB,0BAA0B;IAC1B,QAAQ,CAAC,MAAM,EAAE,SAAS,OAAO,EAAE,CAAC;IACpC,mGAAmG;IACnG,QAAQ,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC;CACpC;AAeD;;;GAGG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,GAAG,IAAI,CAUzD;AAED;;;;;GAKG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,GAAG,MAAM,CAE1E;AAaD;;;GAGG;AACH,wBAAgB,SAAS,CAAC,MAAM,EAAE,SAAS,SAAS,EAAE,EAAE,KAAK,EAAE,SAAS,SAAS,EAAE,GAAG,SAAS,CAgB9F;AAED;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,SAAS,SAAS,EAAE,EAAE,KAAK,EAAE,SAAS,SAAS,EAAE,GAAG,SAAS,MAAM,EAAE,CAMhH;AAED,gHAAgH;AAChH,wBAAgB,qBAAqB,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,CAE7E;AAED,wBAAgB,eAAe,CAAC,IAAI,EAAE,OAAO,GAAG,MAAM,CAErD;AAED,oGAAoG;AACpG,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,SAAS,OAAO,EAAE,EAAE,KAAK,SAAI,GAAG,MAAM,CAI7E"}
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The CPU a run's tree used while its idle bound counted down. A test that runs a compiler is a build inside the test,
|
|
3
|
+
* which is a defect and not a test (specs/cowshed/08_testing.md, "Integration tests (real substrate)"): the pre-signal
|
|
4
|
+
* snapshot names the processes whose CPU time rose between its two process censuses, so the expiry fails as that and
|
|
5
|
+
* not as an anonymous hang. CPU never restarts the idle bound, which counts only finished tests and the progress a
|
|
6
|
+
* running test reports (progress-channel.ts); this says what ran.
|
|
7
|
+
*/
|
|
8
|
+
/** What a rising process makes of the expiry, said in the same words by the snapshot and by the verdict. */
|
|
9
|
+
export const CPU_RISING_CLASS = "a process in the test's tree was using CPU: likely a build inside the test";
|
|
10
|
+
/**
|
|
11
|
+
* The least gain that names a rise, whatever resolution ps prints CPU time at: 10% of a core over the snapshot's five
|
|
12
|
+
* seconds between censuses.
|
|
13
|
+
*/
|
|
14
|
+
export const CPU_RISE_FLOOR_MS = 500;
|
|
15
|
+
/** The longest command a rise keeps: a compiler's argument list runs to kilobytes, and its name leads it. */
|
|
16
|
+
const COMMAND_CHARS = 160;
|
|
17
|
+
function resolutionOfFraction(digits) {
|
|
18
|
+
switch (digits) {
|
|
19
|
+
case 0:
|
|
20
|
+
return 1_000;
|
|
21
|
+
case 1:
|
|
22
|
+
return 100;
|
|
23
|
+
case 2:
|
|
24
|
+
return 10;
|
|
25
|
+
default:
|
|
26
|
+
return 1;
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* A ps CPU time (`cputime=`): `[[D-]HH:]MM:SS` with an optional fraction, as Linux prints it (whole seconds) and macOS
|
|
31
|
+
* does (centiseconds). Null for anything else, such as the `-` ps prints for a process it may not read.
|
|
32
|
+
*/
|
|
33
|
+
export function parseCpuTime(text) {
|
|
34
|
+
const match = /^(?:(\d+)-)?(?:(\d+):)?(\d+):(\d+(?:\.(\d+))?)$/.exec(text);
|
|
35
|
+
if (match === null)
|
|
36
|
+
return null;
|
|
37
|
+
const [, days, hours, minutes, seconds, fraction] = match;
|
|
38
|
+
if (minutes === undefined || seconds === undefined)
|
|
39
|
+
return null;
|
|
40
|
+
const wholeMinutes = ((Number(days ?? 0) * 24 + Number(hours ?? 0)) * 60 + Number(minutes)) * 60;
|
|
41
|
+
return {
|
|
42
|
+
ms: Math.round((wholeMinutes + Number(seconds)) * 1_000),
|
|
43
|
+
resolutionMs: resolutionOfFraction(fraction?.length ?? 0),
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* The least gain between two readings of one process that names a rise: twice the coarser of their resolutions, and
|
|
48
|
+
* never under {@link CPU_RISE_FLOOR_MS}. A process that crossed a boundary of the last printed digit on a few
|
|
49
|
+
* milliseconds of work gains one resolution, so two are more than one. That is 2,000 ms between whole-second readings
|
|
50
|
+
* and 500 ms between centisecond ones.
|
|
51
|
+
*/
|
|
52
|
+
export function cpuRiseThresholdMs(before, after) {
|
|
53
|
+
return Math.max(CPU_RISE_FLOOR_MS, 2 * Math.max(before.resolutionMs, after.resolutionMs));
|
|
54
|
+
}
|
|
55
|
+
/** Each process present in both censuses (same pid and start), as its earlier and its later sample. */
|
|
56
|
+
function paired(before, after) {
|
|
57
|
+
const earlier = new Map(before.map((sample) => [`${sample.pid} ${sample.started}`, sample]));
|
|
58
|
+
const pairs = [];
|
|
59
|
+
for (const sample of after) {
|
|
60
|
+
const first = earlier.get(`${sample.pid} ${sample.started}`);
|
|
61
|
+
if (first !== undefined)
|
|
62
|
+
pairs.push([first, sample]);
|
|
63
|
+
}
|
|
64
|
+
return pairs;
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* The processes present in both censuses (same pid and start) whose CPU time rose by at least
|
|
68
|
+
* {@link cpuRiseThresholdMs}.
|
|
69
|
+
*/
|
|
70
|
+
export function cpuChange(before, after) {
|
|
71
|
+
const rising = [];
|
|
72
|
+
const unread = [];
|
|
73
|
+
for (const [first, sample] of paired(before, after)) {
|
|
74
|
+
if (first.cpu === null || sample.cpu === null) {
|
|
75
|
+
unread.push(sample.pid);
|
|
76
|
+
continue;
|
|
77
|
+
}
|
|
78
|
+
const gainedMs = sample.cpu.ms - first.cpu.ms;
|
|
79
|
+
if (gainedMs < cpuRiseThresholdMs(first.cpu, sample.cpu))
|
|
80
|
+
continue;
|
|
81
|
+
const command = sample.command.length > COMMAND_CHARS ? `${sample.command.slice(0, COMMAND_CHARS)}…` : sample.command;
|
|
82
|
+
rising.push({ pid: sample.pid, command, gainedMs });
|
|
83
|
+
}
|
|
84
|
+
rising.sort((a, b) => b.gainedMs - a.gainedMs || a.pid - b.pid);
|
|
85
|
+
return { rising, unread };
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* The gains {@link cpuChange} held these censuses to, each distinct one once, smallest first: one per resolution of the
|
|
89
|
+
* readings compared, and none when no process had a readable CPU time in both.
|
|
90
|
+
*/
|
|
91
|
+
export function cpuRiseThresholdsMs(before, after) {
|
|
92
|
+
const thresholds = new Set();
|
|
93
|
+
for (const [first, sample] of paired(before, after)) {
|
|
94
|
+
if (first.cpu !== null && sample.cpu !== null)
|
|
95
|
+
thresholds.add(cpuRiseThresholdMs(first.cpu, sample.cpu));
|
|
96
|
+
}
|
|
97
|
+
return [...thresholds].sort((a, b) => a - b);
|
|
98
|
+
}
|
|
99
|
+
/** The gains of {@link cpuRiseThresholdsMs} in seconds, as the snapshot says them: `2 s`, or `0.5 s or 2 s`. */
|
|
100
|
+
export function describeCpuThresholds(thresholdsMs) {
|
|
101
|
+
return thresholdsMs.map((ms) => `${ms / 1_000} s`).join(' or ');
|
|
102
|
+
}
|
|
103
|
+
export function describeCpuRise(rise) {
|
|
104
|
+
return `pid ${rise.pid} (${rise.command}) gained ${(rise.gainedMs / 1_000).toFixed(1)} s of CPU`;
|
|
105
|
+
}
|
|
106
|
+
/** The largest gains, named, and how many more there were: a parallel build runs many processes. */
|
|
107
|
+
export function describeCpuRises(rises, named = 3) {
|
|
108
|
+
const shown = rises.slice(0, named).map(describeCpuRise);
|
|
109
|
+
const more = rises.length - shown.length;
|
|
110
|
+
return `${shown.join('; ')}${more > 0 ? `; and ${more} more` : ''}`;
|
|
111
|
+
}
|
|
@@ -4,7 +4,7 @@ import { type RunOutcome } from './memory-retry.js';
|
|
|
4
4
|
import { type SubtreeOwner } from './owned-subtree.js';
|
|
5
5
|
import { type InheritedCpuPool } from './pool-environment.js';
|
|
6
6
|
import { RamTempVolume } from './ram-temp.js';
|
|
7
|
-
import type
|
|
7
|
+
import { type BoundedExecOptions } from './schema.js';
|
|
8
8
|
import { type BoundedExecTask } from './verdict.js';
|
|
9
9
|
export interface BoundedExecContext {
|
|
10
10
|
root: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/executor.ts"],"names":[],"mappings":"AAeA,OAAO,EAAwB,KAAK,SAAS,EAAE,MAAM,sBAAsB,CAAC;
|
|
1
|
+
{"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/executor.ts"],"names":[],"mappings":"AAeA,OAAO,EAAwB,KAAK,SAAS,EAAE,MAAM,sBAAsB,CAAC;AAO5E,OAAO,EAAE,sBAAsB,EAAmB,MAAM,uBAAuB,CAAC;AAahF,OAAO,EAAqB,KAAK,UAAU,EAA8B,MAAM,mBAAmB,CAAC;AAEnG,OAAO,EAOL,KAAK,YAAY,EAClB,MAAM,oBAAoB,CAAC;AAG5B,OAAO,EAAgB,KAAK,gBAAgB,EAAwB,MAAM,uBAAuB,CAAC;AAGlG,OAAO,EASL,aAAa,EAEd,MAAM,eAAe,CAAC;AAIvB,OAAO,EAAE,KAAK,kBAAkB,EAAkB,MAAM,aAAa,CAAC;AAEtE,OAAO,EAEL,KAAK,eAAe,EAcrB,MAAM,cAAc,CAAC;AAwBtB,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,wFAAwF;IACxF,MAAM,CAAC,EAAE;QAAE,IAAI,EAAE,eAAe,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC;IACtD,uFAAuF;IACvF,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;CAClC;AAED,qEAAqE;AACrE,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,SAAS,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE;YAAE,IAAI,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC;CAC1D;AAED,MAAM,MAAM,iBAAiB,GAAG,UAAU,CAAC;AAoB3C,MAAM,WAAW,iBAAiB;IAChC,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1D;AAED,8EAA8E;AAC9E,MAAM,MAAM,UAAU,GAAG,IAAI,CAC3B,aAAa,EACb,SAAS,GAAG,SAAS,GAAG,UAAU,GAAG,OAAO,GAAG,QAAQ,GAAG,gBAAgB,CAC3E,CAAC;AAQF;;;;;;GAMG;AACH,wBAAgB,wBAAwB,CACtC,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,MAAM,GAAG,SAAS,EACzB,SAAS,EAAE,MAAM,GAAG,SAAS,EAC7B,SAAS,EAAE,MAAM,GAAG,SAAS,EAC7B,GAAG,EAAE,MAAM,EACX,MAAM,GAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAOzB,GACA,MAAM,GAAG,SAAS,CAiBpB;AAED,MAAM,CAAC,OAAO,UAAU,mBAAmB,CACzC,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,aAAa,GACrB,OAAO,CAAC,iBAAiB,CAAC,CAE5B;AAED;;;GAGG;AACH,wBAAgB,qBAAqB,CACnC,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,kBAAkB,GAC1B,OAAO,CAAC,iBAAiB,CAAC,CAc5B;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAChC,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,kBAAkB,EAC3B,MAAM,EAAE,CAAC,OAAO,EAAE,MAAM,GAAG,SAAS,KAAK,UAAU,GAAG,IAAI,GACzD,OAAO,CAAC,iBAAiB,CAAC,CAsB5B;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAAC,OAAO,EAAE,aAAa,GAAG,kBAAkB,CAiBxE;AAED;;;;;;;;GAQG;AACH,wBAAsB,cAAc,CAClC,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,kBAAkB,EAC3B,MAAM,EAAE,iBAAiB,EACzB,UAAU,EAAE,UAAU,GAAG,IAAI,EAC7B,SAAS,GAAE,gBAAgB,GAAG,IAAW,EACzC,kBAAkB,GAAE,OAAO,sBAA+C,EAC1E,KAAK,GAAE,YAAY,GAAG,IAAW,EACjC,SAAS,CAAC,EAAE,SAAS,GACpB,OAAO,CAAC,iBAAiB,CAAC,CAe5B;AA+oCD,wBAAgB,uBAAuB,IAAI,iBAAiB,CAM3D"}
|
|
@@ -23,10 +23,12 @@ import { askParentPool, describeGrant } from './parent-grant.js';
|
|
|
23
23
|
import { readLastPeak } from './peak-charge.js';
|
|
24
24
|
import { CPU_POOL_ENV, readInheritedCpuPool } from './pool-environment.js';
|
|
25
25
|
import { createRunProgress, withProgressReport, withTestResultReport } from './progress.js';
|
|
26
|
+
import { PROGRESS_SOCKET_ENV, serveProgressChannel } from './progress-channel.js';
|
|
26
27
|
import { createHostCommands, describeCoverageGap, describeRamTempError, describeSignalled, LEFTOVER_GRACE_MS, PRODUCTION, RAM_TEMP_CAPACITY_BYTES, RamTempVolume, ramTempPaths, } from './ram-temp.js';
|
|
27
28
|
import { createRunnerLifecycle, withFixtureLifecycle, withRunnerLifecycle } from './runner-lifecycle.js';
|
|
28
29
|
import { CPU_TOKENS_ENV, runnerDemand, sizeRunner, sizeRunnerToGrant } from './runner-sizing.js';
|
|
29
30
|
import { checkRuntimePin, describeRuntimePinRefusal } from './runtime-pin.js';
|
|
31
|
+
import { boundedCommand } from './schema.js';
|
|
30
32
|
import { createTestResultWriter } from './test-result-writer.js';
|
|
31
33
|
import { BUN_TEST, clearTaskDirectory, describeNoSpace, JUNIT_FILE, JUNIT_REPORT_ENV, judgeOperationalFailure, judgeRun, readJunitReport, taskDirectory, withJunitReport, writeRecord, } from './verdict.js';
|
|
32
34
|
const DEFAULT_KILL_AFTER_MS = 10_000;
|
|
@@ -173,7 +175,7 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
173
175
|
if (directory !== null) {
|
|
174
176
|
await clearTaskDirectory(directory);
|
|
175
177
|
}
|
|
176
|
-
const requested =
|
|
178
|
+
const requested = boundedCommand(options);
|
|
177
179
|
const timeoutMs = options.timeoutMs;
|
|
178
180
|
const idleTimeoutMs = options.idleTimeoutMs;
|
|
179
181
|
const killAfterMs = options.killAfterMs ?? DEFAULT_KILL_AFTER_MS;
|
|
@@ -192,6 +194,7 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
192
194
|
retireError: null,
|
|
193
195
|
noSpace: null,
|
|
194
196
|
lockWaits: [],
|
|
197
|
+
cpuRising: [],
|
|
195
198
|
};
|
|
196
199
|
const appendStdout = (chunk) => {
|
|
197
200
|
const text = chunk.toString();
|
|
@@ -429,6 +432,18 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
429
432
|
let pool = null;
|
|
430
433
|
let reservation = null;
|
|
431
434
|
const releaseReservation = () => reservation?.release();
|
|
435
|
+
// The run's progress channel (progress-channel.ts), served when the run has an idle bound for its records to restart,
|
|
436
|
+
// and closed once its command is gone. A channel that leaves its socket behind says so; the run's verdict stands.
|
|
437
|
+
let progressChannel = null;
|
|
438
|
+
let progressClosed = null;
|
|
439
|
+
const closeProgress = () => {
|
|
440
|
+
progressClosed ??= (async () => {
|
|
441
|
+
const left = (await progressChannel?.close()) ?? null;
|
|
442
|
+
if (left !== null)
|
|
443
|
+
appendStderr(`bounded-exec: ${left}\n`);
|
|
444
|
+
})();
|
|
445
|
+
return progressClosed;
|
|
446
|
+
};
|
|
432
447
|
let cpuReleased = null;
|
|
433
448
|
const releaseCpu = () => {
|
|
434
449
|
cpuReleased ??= (async () => {
|
|
@@ -456,7 +471,7 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
456
471
|
});
|
|
457
472
|
// Records the run's verdict when it has a task to record as. An `expiry` is set only when a bound ended the run.
|
|
458
473
|
// Observation and ownership failures remain operational even after valid queue evidence; cancellation alone does not.
|
|
459
|
-
const recordRun = async (exitCode, expiry, elapsedMs, operational, noSpace, lockWaits, peakBytes) => {
|
|
474
|
+
const recordRun = async (exitCode, expiry, elapsedMs, operational, noSpace, lockWaits, cpuRising, peakBytes) => {
|
|
460
475
|
if (context.record === undefined) {
|
|
461
476
|
return;
|
|
462
477
|
}
|
|
@@ -486,6 +501,7 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
486
501
|
report,
|
|
487
502
|
noSpace,
|
|
488
503
|
lockWaits,
|
|
504
|
+
cpuRising,
|
|
489
505
|
admissionQueue,
|
|
490
506
|
});
|
|
491
507
|
if (verdict.outcome === 'bound' && verdict.bound === 'test') {
|
|
@@ -504,9 +520,10 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
504
520
|
const refuse = async (message) => {
|
|
505
521
|
appendStderr(message);
|
|
506
522
|
await releaseCpu();
|
|
523
|
+
await closeProgress();
|
|
507
524
|
// The command never started, so its lease holds nothing of its own to keep.
|
|
508
525
|
await releaseLease(false);
|
|
509
|
-
await recordRun(1, null, Date.now() - asked, false, null, [], null);
|
|
526
|
+
await recordRun(1, null, Date.now() - asked, false, null, [], [], null);
|
|
510
527
|
return { success: false, terminalOutput: outputChunks.join('') };
|
|
511
528
|
};
|
|
512
529
|
if (parentRefusal !== null) {
|
|
@@ -529,6 +546,29 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
529
546
|
}
|
|
530
547
|
pool = created.pool;
|
|
531
548
|
}
|
|
549
|
+
// A running test reports its own progress on the run's progress channel, and each record restarts the idle bound as a
|
|
550
|
+
// finished test does; a record that is not one fails the run as a broken reporter. A run with no idle bound has
|
|
551
|
+
// nothing for a record to restart and is served none. Only the command learns the socket, from its environment, so
|
|
552
|
+
// these handlers run after it is spawned and its bounds are wired below.
|
|
553
|
+
if (idleTimeoutMs !== undefined) {
|
|
554
|
+
const served = await serveProgressChannel({
|
|
555
|
+
moved: () => {
|
|
556
|
+
if (fixtureDone)
|
|
557
|
+
stepTo({ kind: 'reported', at: Date.now(), testing: isTesting() });
|
|
558
|
+
},
|
|
559
|
+
invalid: (detail) => {
|
|
560
|
+
if (lifecycleFailure !== null || state.expiry !== null || state.settled)
|
|
561
|
+
return;
|
|
562
|
+
lifecycleFailure = `progress channel: ${detail}`;
|
|
563
|
+
appendStderr(`bounded-exec: ${lifecycleFailure}\n`);
|
|
564
|
+
termination = terminate();
|
|
565
|
+
},
|
|
566
|
+
});
|
|
567
|
+
if (!served.ok) {
|
|
568
|
+
return refuse(`bounded-exec: the run's progress channel could not be served (${served.error}); refusing to run the command\n`);
|
|
569
|
+
}
|
|
570
|
+
progressChannel = served.channel;
|
|
571
|
+
}
|
|
532
572
|
const native = directory === null ? { command: sized.command, env: {} } : withTestResultReport(sized.command);
|
|
533
573
|
const command = directory === null
|
|
534
574
|
? withProgressReport(native.command)
|
|
@@ -545,6 +585,7 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
545
585
|
...(pool?.environment ?? {}),
|
|
546
586
|
...(await grantMemoryEnvironment(grant?.charged ?? null, process.env)),
|
|
547
587
|
...(leaf?.kind === 'cargo' ? { MBX_GRANT_FD: String(MBX_GRANT_FD) } : {}),
|
|
588
|
+
...(progressChannel === null ? {} : { [PROGRESS_SOCKET_ENV]: progressChannel.socket }),
|
|
548
589
|
};
|
|
549
590
|
// A fixture rides as segment 0 of the lifecycle. Without the channel (Windows) it is plain shell text ahead
|
|
550
591
|
// of the command: the progress bound runs during it and its own ceiling does not. Under mbx a leaf that names its
|
|
@@ -644,7 +685,8 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
644
685
|
return answer;
|
|
645
686
|
});
|
|
646
687
|
}
|
|
647
|
-
// Progress is a finished test,
|
|
688
|
+
// Progress in the output is a finished test, never output itself: a runner's SLOW warnings and a hung test's heartbeat
|
|
689
|
+
// keep printing. A running test reports its own progress on the progress channel instead.
|
|
648
690
|
const progress = createRunProgress(command, resultWriter === null
|
|
649
691
|
? undefined
|
|
650
692
|
: (result) => {
|
|
@@ -881,10 +923,11 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
881
923
|
queue.stop();
|
|
882
924
|
return killChildTree();
|
|
883
925
|
})
|
|
926
|
+
.finally(closeProgress)
|
|
884
927
|
.finally(releaseCpu)
|
|
885
928
|
.finally(() => releaseLease(true))
|
|
886
929
|
.finally(releaseReservation)
|
|
887
|
-
.finally(async () => recordRun(signalToExitCode(signal), state.expiry, Date.now() - startedAt, state.poolCloseError !== null || lifecycleFailure !== null || state.retireError !== null, state.noSpace === null ? null : await state.noSpace, state.lockWaits, null))
|
|
930
|
+
.finally(async () => recordRun(signalToExitCode(signal), state.expiry, Date.now() - startedAt, state.poolCloseError !== null || lifecycleFailure !== null || state.retireError !== null, state.noSpace === null ? null : await state.noSpace, state.lockWaits, state.cpuRising, null))
|
|
888
931
|
.finally(() => process.kill(process.pid, signal));
|
|
889
932
|
};
|
|
890
933
|
const onSigint = () => onTerminationSignal('SIGINT');
|
|
@@ -930,7 +973,7 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
930
973
|
clearTimeout(boundTimer);
|
|
931
974
|
const elapsedMs = Date.now() - startedAt;
|
|
932
975
|
appendStderr(bound.kind === 'idle'
|
|
933
|
-
? `\nCommand made no progress: no test finished for ${bound.limitMs}ms (idleTimeoutMs=${bound.limitMs}) after ${elapsedMs}ms of runtime; ${bound.tests.length > 0
|
|
976
|
+
? `\nCommand made no progress: no test finished or reported progress for ${bound.limitMs}ms (idleTimeoutMs=${bound.limitMs}) after ${elapsedMs}ms of runtime; ${bound.tests.length > 0
|
|
934
977
|
? `still running: ${bound.tests.join(', ')}`
|
|
935
978
|
: 'the runner named no test still running'} (cwd=${cwd}): ${command}\n${queue.unconfirmed === null
|
|
936
979
|
? ''
|
|
@@ -941,6 +984,7 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
941
984
|
if (bound.kind === 'idle') {
|
|
942
985
|
const observed = idleCapture?.result;
|
|
943
986
|
state.lockWaits = observed?.kind === 'captured' ? observed.snapshot.lockWaits : [];
|
|
987
|
+
state.cpuRising = observed?.kind === 'captured' ? observed.snapshot.cpuRising : [];
|
|
944
988
|
appendStderr(`Hang diagnostics: runner pid=${runner ?? 'unavailable'} pgid=${process.platform === 'win32' ? 'unavailable' : (child.pid ?? 'unavailable')} expiryAtMs=${Date.now()} scheduledExpiryAtMs=${idleCapture?.expiresAtMs ?? 'unavailable'}\n`);
|
|
945
989
|
if (observed?.kind === 'captured') {
|
|
946
990
|
appendStderr(`Hang diagnostics: outcome=${observed.snapshot.outcome} capturedAtMs=${observed.snapshot.capturedAtMs} actualExpiryOffsetMs=${Date.now() - observed.snapshot.capturedAtMs}\n${observed.snapshot.text}`);
|
|
@@ -1132,8 +1176,9 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
1132
1176
|
clearTimeout(boundTimer);
|
|
1133
1177
|
cancelCapture();
|
|
1134
1178
|
removeSignalHandlers();
|
|
1135
|
-
// The runner is gone
|
|
1136
|
-
// then the grant goes back to its parent or the host.
|
|
1179
|
+
// The runner is gone, and with it every test that could report on the run's progress channel. Its sub-pool closes
|
|
1180
|
+
// first, so no descendant still draws on tokens the grant returns; then the grant goes back to its parent or the host.
|
|
1181
|
+
await closeProgress();
|
|
1137
1182
|
await releaseCpu();
|
|
1138
1183
|
const code = exitCode ?? signalToExitCode(exitSignal);
|
|
1139
1184
|
if (code !== 0 && state.expiry === null) {
|
|
@@ -1173,7 +1218,7 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
1173
1218
|
// Broken resource ownership, a lost native lifecycle, or leftovers that could not be retired fail even if a shell
|
|
1174
1219
|
// branch later exited 0: its verdict is an ordinary operational failure, never a successful or silently unbounded run.
|
|
1175
1220
|
const operationalFailure = state.poolCloseError !== null || lifecycleFailure !== null || state.retireError !== null;
|
|
1176
|
-
await recordRun(code, state.expiry, Date.now() - startedAt, operationalFailure, seen, state.lockWaits, peakBytes);
|
|
1221
|
+
await recordRun(code, state.expiry, Date.now() - startedAt, operationalFailure, seen, state.lockWaits, state.cpuRising, peakBytes);
|
|
1177
1222
|
const killedForMemory = killedFromOutside({
|
|
1178
1223
|
sentSignal: state.sentSignal,
|
|
1179
1224
|
bounded: state.expiry !== null,
|
|
@@ -1199,19 +1244,6 @@ function resolveCwd(cwd, root) {
|
|
|
1199
1244
|
}
|
|
1200
1245
|
return isAbsolute(cwd) ? cwd : join(root, cwd);
|
|
1201
1246
|
}
|
|
1202
|
-
function buildCommand(options) {
|
|
1203
|
-
const parts = [options.command];
|
|
1204
|
-
if (Array.isArray(options.args)) {
|
|
1205
|
-
parts.push(...options.args);
|
|
1206
|
-
}
|
|
1207
|
-
else if (options.args) {
|
|
1208
|
-
parts.push(options.args);
|
|
1209
|
-
}
|
|
1210
|
-
if (options.forwardAllArgs !== false && options.__unparsed__?.length) {
|
|
1211
|
-
parts.push(...options.__unparsed__);
|
|
1212
|
-
}
|
|
1213
|
-
return parts.join(' ');
|
|
1214
|
-
}
|
|
1215
1247
|
/** Owned scratch is the command's TMPDIR unless the target names one itself. */
|
|
1216
1248
|
function mergeEnv(directory, env) {
|
|
1217
1249
|
const inherited = withoutPoolEnv(process.env);
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { type CpuRise } from './cpu-rise.js';
|
|
1
2
|
import { type LockWait } from './lock-wait.js';
|
|
2
3
|
/**
|
|
3
4
|
* The pre-expiry capture window. Stack samples set it: under load averages of 27-52 on 18 cores one `sudo -n sample`
|
|
@@ -12,6 +13,11 @@ export interface HangDiagnostics {
|
|
|
12
13
|
outcome: 'holders' | 'none' | 'incomplete';
|
|
13
14
|
/** Lock waits the snapshot names: empty unless the capture is complete, and always empty without the open-file census. */
|
|
14
15
|
lockWaits: readonly LockWait[];
|
|
16
|
+
/**
|
|
17
|
+
* The processes of the tree whose CPU time rose between the two censuses, largest gain first: empty when none did, and
|
|
18
|
+
* when the closing census did not complete. A build inside the test, never a restart of the idle bound.
|
|
19
|
+
*/
|
|
20
|
+
cpuRising: readonly CpuRise[];
|
|
15
21
|
text: string;
|
|
16
22
|
}
|
|
17
23
|
export interface HangCaptureRequest {
|
|
@@ -36,6 +42,8 @@ export type HangProbeResult = {
|
|
|
36
42
|
export interface HangDiagnosticHost {
|
|
37
43
|
platform: NodeJS.Platform;
|
|
38
44
|
probe(file: string, args: readonly string[], timeoutMs: number, signal: AbortSignal): Promise<HangProbeResult>;
|
|
45
|
+
/** Waits `ms`, or until `signal` aborts: the capture's end ends its own CPU interval. */
|
|
46
|
+
sleep(ms: number, signal: AbortSignal): Promise<void>;
|
|
39
47
|
}
|
|
40
48
|
/** A pre-expiry observation, never an assertion that an open file is a contested lock. */
|
|
41
49
|
export declare function captureHangDiagnostics(request: HangCaptureRequest, source?: HangDiagnosticHost): Promise<HangDiagnostics>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"hang-diagnostics.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/hang-diagnostics.ts"],"names":[],"mappings":"AAGA,OAAO,EAEL,KAAK,QAAQ,EAKd,MAAM,gBAAgB,CAAC;AAExB;;;GAGG;AACH,eAAO,MAAM,eAAe,QAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"hang-diagnostics.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/hang-diagnostics.ts"],"names":[],"mappings":"AAGA,OAAO,EAEL,KAAK,OAAO,EAOb,MAAM,eAAe,CAAC;AACvB,OAAO,EAEL,KAAK,QAAQ,EAKd,MAAM,gBAAgB,CAAC;AAExB;;;GAGG;AACH,eAAO,MAAM,eAAe,QAAS,CAAC;AA4CtC,MAAM,WAAW,eAAe;IAC9B,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,sEAAsE;IACtE,kBAAkB,EAAE,MAAM,CAAC;IAC3B,OAAO,EAAE,SAAS,GAAG,MAAM,GAAG,YAAY,CAAC;IAC3C,0HAA0H;IAC1H,SAAS,EAAE,SAAS,QAAQ,EAAE,CAAC;IAC/B;;;OAGG;IACH,SAAS,EAAE,SAAS,OAAO,EAAE,CAAC;IAC9B,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,kBAAkB;IACjC,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAED,MAAM,MAAM,eAAe,GACvB;IAAE,MAAM,EAAE,QAAQ,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAClE;IAAE,MAAM,EAAE,aAAa,GAAG,WAAW,GAAG,QAAQ,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC;AAEvG,qGAAqG;AACrG,MAAM,WAAW,kBAAkB;IACjC,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC;IAC1B,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,MAAM,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;IAC/G,yFAAyF;IACzF,KAAK,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACvD;AAwBD,0FAA0F;AAC1F,wBAAgB,sBAAsB,CACpC,OAAO,EAAE,kBAAkB,EAC3B,MAAM,GAAE,kBAAyB,GAChC,OAAO,CAAC,eAAe,CAAC,CA2T1B;AAED;;;;;;;;GAQG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAM5D;AAED,2GAA2G;AAC3G,wBAAgB,QAAQ,CACtB,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,SAAS,MAAM,EAAE,EACvB,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,WAAW,EACnB,WAAW,CAAC,EAAE,MAAM,CAAC,UAAU,GAC9B,OAAO,CAAC,eAAe,CAAC,CA6D1B"}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { spawn } from 'node:child_process';
|
|
2
2
|
import { describeTeardown, killProcessGroupNow, TEARDOWN_PATIENCE } from '../../process-group.js';
|
|
3
3
|
import { withDescendants } from '../../testing.js';
|
|
4
|
+
import { CPU_RISING_CLASS, cpuChange, cpuRiseThresholdsMs, describeCpuRise, describeCpuThresholds, parseCpuTime, } from './cpu-rise.js';
|
|
4
5
|
import { describeHolders, lockCandidates, lockWaits, parseLsofFields, parseLsofTable, } from './lock-wait.js';
|
|
5
6
|
/**
|
|
6
7
|
* The pre-expiry capture window. Stack samples set it: under load averages of 27-52 on 18 cores one `sudo -n sample`
|
|
@@ -8,7 +9,15 @@ import { describeHolders, lockCandidates, lockWaits, parseLsofFields, parseLsofT
|
|
|
8
9
|
*/
|
|
9
10
|
export const HANG_CAPTURE_MS = 10_000;
|
|
10
11
|
const MAX_PROBE_BYTES = 4 * 1024 * 1024;
|
|
11
|
-
const PS_FIELDS = 'pid=,ppid=,pgid=,stat=,%cpu=,etime=,lstart=,command=';
|
|
12
|
+
const PS_FIELDS = 'pid=,ppid=,pgid=,stat=,%cpu=,etime=,cputime=,lstart=,command=';
|
|
13
|
+
/**
|
|
14
|
+
* The least time between the two process censuses, which are the snapshot's two CPU samples. A process is named
|
|
15
|
+
* after gaining 2 s where ps counts whole seconds and 0.5 s where it counts centiseconds (cpuRiseThresholdMs), so five
|
|
16
|
+
* seconds show a process at 40% or at 10% of a core. On Darwin the stack samples have usually spent most of it;
|
|
17
|
+
* elsewhere, or when sudo refuses them, the snapshot pauses, inside its window and early enough that the closing
|
|
18
|
+
* census keeps its own limit.
|
|
19
|
+
*/
|
|
20
|
+
const CPU_INTERVAL_MS = 5_000;
|
|
12
21
|
/**
|
|
13
22
|
* Stack samples: `sample` suspends a process every millisecond for one second and reports its threads' call graph. The
|
|
14
23
|
* runner and at most this many leaf processes are sampled, all at once. `-mayDie` is not passed: it reads symbols
|
|
@@ -38,7 +47,20 @@ function callGraph(report) {
|
|
|
38
47
|
const end = report.indexOf('\nBinary Images:');
|
|
39
48
|
return (end === -1 ? report : report.slice(0, end)).trim();
|
|
40
49
|
}
|
|
41
|
-
const host = { platform: process.platform, probe: runProbe };
|
|
50
|
+
const host = { platform: process.platform, probe: runProbe, sleep: sleepUnlessAborted };
|
|
51
|
+
function sleepUnlessAborted(ms, signal) {
|
|
52
|
+
return new Promise((resolve) => {
|
|
53
|
+
const done = () => {
|
|
54
|
+
clearTimeout(timer);
|
|
55
|
+
signal.removeEventListener('abort', done);
|
|
56
|
+
resolve();
|
|
57
|
+
};
|
|
58
|
+
const timer = setTimeout(done, ms);
|
|
59
|
+
signal.addEventListener('abort', done, { once: true });
|
|
60
|
+
if (signal.aborted)
|
|
61
|
+
done();
|
|
62
|
+
});
|
|
63
|
+
}
|
|
42
64
|
/** A pre-expiry observation, never an assertion that an open file is a contested lock. */
|
|
43
65
|
export function captureHangDiagnostics(request, source = host) {
|
|
44
66
|
const startedAtMs = Date.now();
|
|
@@ -53,6 +75,7 @@ export function captureHangDiagnostics(request, source = host) {
|
|
|
53
75
|
const during = () => (active.size === 0 ? 'process census' : [...active].join(', '));
|
|
54
76
|
let finished = false;
|
|
55
77
|
let waits = [];
|
|
78
|
+
let rising = [];
|
|
56
79
|
// This package targets lib es2022, which does not declare Promise.withResolvers.
|
|
57
80
|
return new Promise((resolve) => {
|
|
58
81
|
const finish = (outcome) => {
|
|
@@ -70,6 +93,7 @@ export function captureHangDiagnostics(request, source = host) {
|
|
|
70
93
|
offsetFromExpiryMs,
|
|
71
94
|
outcome,
|
|
72
95
|
lockWaits: outcome === 'incomplete' ? [] : waits,
|
|
96
|
+
cpuRising: rising,
|
|
73
97
|
text: `${lines.join('\n')}\nHang snapshot outcome=${outcome} capturedAtMs=${capturedAtMs} offsetFromExpiryMs=${offsetFromExpiryMs}\n`,
|
|
74
98
|
});
|
|
75
99
|
};
|
|
@@ -116,27 +140,37 @@ export function captureHangDiagnostics(request, source = host) {
|
|
|
116
140
|
}
|
|
117
141
|
const parsed = [];
|
|
118
142
|
for (const row of result.stdout.split('\n').filter((line) => line.trim() !== '')) {
|
|
119
|
-
const match = /^\s*(\d+)\s+(\d+)\s+(\d+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+\s+\S+\s+\d+\s+\d\d:\d\d:\d\d\s+\d{4})\s+(.*)$/.exec(row);
|
|
143
|
+
const match = /^\s*(\d+)\s+(\d+)\s+(\d+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+\s+\S+\s+\d+\s+\d\d:\d\d:\d\d\s+\d{4})\s+(.*)$/.exec(row);
|
|
120
144
|
if (match === null) {
|
|
121
145
|
lines.push(`${name}: could not decode ps row: ${row}`);
|
|
122
146
|
return null;
|
|
123
147
|
}
|
|
124
|
-
const [, pid, ppid, pgid, stat, , , started, command] = match;
|
|
148
|
+
const [, pid, ppid, pgid, stat, , , cputime, started, command] = match;
|
|
125
149
|
if (pid === undefined ||
|
|
126
150
|
ppid === undefined ||
|
|
127
151
|
pgid === undefined ||
|
|
128
152
|
stat === undefined ||
|
|
153
|
+
cputime === undefined ||
|
|
129
154
|
started === undefined ||
|
|
130
155
|
command === undefined) {
|
|
131
156
|
lines.push(`${name}: incomplete ps row: ${row}`);
|
|
132
157
|
return null;
|
|
133
158
|
}
|
|
134
|
-
parsed.push({
|
|
159
|
+
parsed.push({
|
|
160
|
+
pid: Number(pid),
|
|
161
|
+
ppid: Number(ppid),
|
|
162
|
+
pgid: Number(pgid),
|
|
163
|
+
stat,
|
|
164
|
+
cpu: parseCpuTime(cputime),
|
|
165
|
+
started,
|
|
166
|
+
command,
|
|
167
|
+
raw: row,
|
|
168
|
+
});
|
|
135
169
|
}
|
|
136
170
|
const roots = [request.pid, ...parsed.filter((entry) => entry.pgid === request.pgid).map((entry) => entry.pid)];
|
|
137
171
|
const selected = new Set(withDescendants(parsed, roots).map((entry) => entry.pid));
|
|
138
172
|
const tree = parsed.filter((entry) => selected.has(entry.pid));
|
|
139
|
-
lines.push(`${name}: PID PPID PGID STAT %CPU ELAPSED STARTED COMMAND`);
|
|
173
|
+
lines.push(`${name}: PID PPID PGID STAT %CPU ELAPSED CPUTIME STARTED COMMAND`);
|
|
140
174
|
lines.push(...tree.map((entry) => entry.raw));
|
|
141
175
|
if (!tree.some((entry) => entry.pid === request.pid)) {
|
|
142
176
|
lines.push(`${name}: root pid ${request.pid} exited or is not visible; capture raced or is incomplete.`);
|
|
@@ -203,6 +237,7 @@ export function captureHangDiagnostics(request, source = host) {
|
|
|
203
237
|
return;
|
|
204
238
|
if (before === null)
|
|
205
239
|
return finish('incomplete');
|
|
240
|
+
const beforeAtMs = Date.now();
|
|
206
241
|
let incomplete = false;
|
|
207
242
|
let observed = before.length > 0;
|
|
208
243
|
if (source.platform === 'darwin') {
|
|
@@ -259,11 +294,20 @@ export function captureHangDiagnostics(request, source = host) {
|
|
|
259
294
|
lines.push(`Open-file census not provided on ${source.platform}; observations below name processes only.`);
|
|
260
295
|
lines.push(`Stack samples not provided on ${source.platform}; no blocking call is observed.`);
|
|
261
296
|
}
|
|
297
|
+
const settleMs = Math.min(beforeAtMs + CPU_INTERVAL_MS, endsAtMs - CENSUS_LIMIT_MS) - Date.now();
|
|
298
|
+
if (settleMs > 0) {
|
|
299
|
+
active.add('CPU interval');
|
|
300
|
+
await source.sleep(settleMs, controller.signal);
|
|
301
|
+
active.delete('CPU interval');
|
|
302
|
+
if (finished)
|
|
303
|
+
return;
|
|
304
|
+
}
|
|
262
305
|
const after = await census('ps after');
|
|
263
306
|
if (finished)
|
|
264
307
|
return;
|
|
265
308
|
if (after === null)
|
|
266
309
|
return finish('incomplete');
|
|
310
|
+
const afterAtMs = Date.now();
|
|
267
311
|
if (before.length !== after.length ||
|
|
268
312
|
before.some((entry) => !after.some((other) => other.pid === entry.pid &&
|
|
269
313
|
other.ppid === entry.ppid &&
|
|
@@ -274,6 +318,22 @@ export function captureHangDiagnostics(request, source = host) {
|
|
|
274
318
|
incomplete = true;
|
|
275
319
|
lines.push('Process identities or membership changed during capture; open-file census raced.');
|
|
276
320
|
}
|
|
321
|
+
const change = cpuChange(before, after);
|
|
322
|
+
const thresholdsMs = cpuRiseThresholdsMs(before, after);
|
|
323
|
+
rising = change.rising;
|
|
324
|
+
const intervalMs = afterAtMs - beforeAtMs;
|
|
325
|
+
if (change.unread.length > 0) {
|
|
326
|
+
lines.push(`CPU time not read for pid ${change.unread.join(', ')}: ps printed none that parses, so they are not counted idle.`);
|
|
327
|
+
}
|
|
328
|
+
if (rising.length > 0) {
|
|
329
|
+
lines.push(`CPU rose in the tree over about ${intervalMs}ms between the censuses (a rise is named from ${describeCpuThresholds(thresholdsMs)} of CPU):`, ...rising.map((rise) => ` ${describeCpuRise(rise)}`), `Classification: ${CPU_RISING_CLASS}`);
|
|
330
|
+
}
|
|
331
|
+
else if (thresholdsMs.length === 0) {
|
|
332
|
+
lines.push('CPU: no process of the tree had a CPU time ps gave in both censuses, so none was compared.');
|
|
333
|
+
}
|
|
334
|
+
else {
|
|
335
|
+
lines.push(`CPU: no process of the tree gained ${describeCpuThresholds(thresholdsMs)} of CPU over about ${intervalMs}ms between the censuses.`);
|
|
336
|
+
}
|
|
277
337
|
if (!incomplete) {
|
|
278
338
|
for (const wait of waits) {
|
|
279
339
|
lines.push(`Lock wait (open-handle evidence, not a kernel lock owner): ${wait.path} open in the run by pid ${wait.waiters.join(', ')} and outside it by ${describeHolders(wait.holders)}`);
|