@norskvideo/ctl-test-harness 0.1.102 → 0.1.104
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/contention/bun-output.d.ts +19 -0
- package/contention/bun-output.js +60 -0
- package/contention/cli.d.ts +9 -0
- package/contention/cli.js +80 -0
- package/contention/index.d.ts +7 -0
- package/contention/index.js +11 -0
- package/contention/psi.d.ts +54 -0
- package/contention/psi.js +119 -0
- package/contention/run.d.ts +50 -0
- package/contention/run.js +187 -0
- package/contention/sampler.d.ts +23 -0
- package/contention/sampler.js +62 -0
- package/contention/summary.d.ts +3 -0
- package/contention/summary.js +48 -0
- package/contention/timeline.d.ts +50 -0
- package/contention/timeline.js +99 -0
- package/contention/verdict.d.ts +117 -0
- package/contention/verdict.js +220 -0
- package/package.json +8 -3
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import type { SuiteFailure } from "./verdict.js";
|
|
2
|
+
export type Stream = "stdout" | "stderr";
|
|
3
|
+
export declare class BunTestOutput {
|
|
4
|
+
private readonly now;
|
|
5
|
+
private readonly partial;
|
|
6
|
+
private readonly found;
|
|
7
|
+
private readonly seen;
|
|
8
|
+
private summary;
|
|
9
|
+
private file;
|
|
10
|
+
constructor(now?: () => number);
|
|
11
|
+
feed(chunk: string, stream?: Stream): void;
|
|
12
|
+
failures(): SuiteFailure[];
|
|
13
|
+
counts(): {
|
|
14
|
+
pass: number;
|
|
15
|
+
fail: number;
|
|
16
|
+
skip: number;
|
|
17
|
+
};
|
|
18
|
+
private line;
|
|
19
|
+
}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
// biome-ignore lint/suspicious/noControlCharactersInRegex: matching ANSI escapes is the point.
|
|
2
|
+
const ANSI = /\u001b\[[0-9;]*m/g;
|
|
3
|
+
const FILE_HEADING = /^(\S.*\.(?:test|spec)\.[cm]?[jt]sx?):$/;
|
|
4
|
+
const FAIL = /^(?:\(fail\)|✗)\s+(.+?)(?:\s+\[(\d+(?:\.\d+)?)(ms|s)\])?$/;
|
|
5
|
+
const PASS = /^(?:\(pass\)|✓)\s+/;
|
|
6
|
+
const SUMMARY = /^(\d+) (pass|fail|skip)$/;
|
|
7
|
+
export class BunTestOutput {
|
|
8
|
+
now;
|
|
9
|
+
partial = { stdout: "", stderr: "" };
|
|
10
|
+
found = [];
|
|
11
|
+
seen = { pass: 0, fail: 0, skip: 0 };
|
|
12
|
+
summary = {};
|
|
13
|
+
file;
|
|
14
|
+
constructor(now = Date.now) {
|
|
15
|
+
this.now = now;
|
|
16
|
+
}
|
|
17
|
+
feed(chunk, stream = "stderr") {
|
|
18
|
+
const lines = (this.partial[stream] + chunk).split("\n");
|
|
19
|
+
this.partial[stream] = lines.pop() ?? "";
|
|
20
|
+
for (const line of lines)
|
|
21
|
+
this.line(line.replace(ANSI, "").trim());
|
|
22
|
+
}
|
|
23
|
+
failures() {
|
|
24
|
+
return [...this.found];
|
|
25
|
+
}
|
|
26
|
+
counts() {
|
|
27
|
+
return {
|
|
28
|
+
pass: this.summary.pass ?? this.seen.pass,
|
|
29
|
+
fail: this.summary.fail ?? this.seen.fail,
|
|
30
|
+
skip: this.summary.skip ?? this.seen.skip,
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
line(text) {
|
|
34
|
+
const heading = FILE_HEADING.exec(text);
|
|
35
|
+
if (heading) {
|
|
36
|
+
this.file = heading[1];
|
|
37
|
+
return;
|
|
38
|
+
}
|
|
39
|
+
const fail = FAIL.exec(text);
|
|
40
|
+
if (fail) {
|
|
41
|
+
const n = fail[2] === undefined ? 0 : Number(fail[2]);
|
|
42
|
+
const durationMs = fail[3] === "s" ? n * 1000 : n;
|
|
43
|
+
this.found.push({
|
|
44
|
+
name: fail[1],
|
|
45
|
+
...(this.file ? { file: this.file } : {}),
|
|
46
|
+
durationMs,
|
|
47
|
+
endMs: this.now(),
|
|
48
|
+
});
|
|
49
|
+
this.seen.fail++;
|
|
50
|
+
return;
|
|
51
|
+
}
|
|
52
|
+
if (PASS.test(text)) {
|
|
53
|
+
this.seen.pass++;
|
|
54
|
+
return;
|
|
55
|
+
}
|
|
56
|
+
const summary = SUMMARY.exec(text);
|
|
57
|
+
if (summary)
|
|
58
|
+
this.summary[summary[2]] = Number(summary[1]);
|
|
59
|
+
}
|
|
60
|
+
}
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
#!/usr/bin/env bun
|
|
2
|
+
// `ctl-contention`, what the dev-kit's integration.yml runs a suite under once
|
|
3
|
+
// a product opts in (scripts/integration/contention.json):
|
|
4
|
+
//
|
|
5
|
+
// ctl-contention run [--config <file>] [--out <dir>] -- <command...>
|
|
6
|
+
// run the command with the pressure sampler alongside; exit 0 on a pass,
|
|
7
|
+
// the command's own status on a real failure, 75 when uncertain
|
|
8
|
+
// ctl-contention wait-quiet [--config <file>] before a rerun: wait for a quiet host
|
|
9
|
+
// ctl-contention config <key> [--config <file>] print one resolved setting (e.g. rerun.max)
|
|
10
|
+
// ctl-contention probe print where pressure can be read, and one sample
|
|
11
|
+
import { detectPressure, readFileText, readSample } from "./psi.js";
|
|
12
|
+
import { loadContentionConfig, runUnderContention, waitForQuiet } from "./run.js";
|
|
13
|
+
export function parseCliArgs(argv) {
|
|
14
|
+
const dash = argv.indexOf("--");
|
|
15
|
+
const head = dash === -1 ? [...argv] : argv.slice(0, dash);
|
|
16
|
+
const command = dash === -1 ? [] : argv.slice(dash + 1);
|
|
17
|
+
const args = { verb: undefined, config: undefined, out: undefined, positional: [], command };
|
|
18
|
+
for (let i = 0; i < head.length; i++) {
|
|
19
|
+
const a = head[i];
|
|
20
|
+
if (a === "--config")
|
|
21
|
+
args.config = head[++i];
|
|
22
|
+
else if (a === "--out")
|
|
23
|
+
args.out = head[++i];
|
|
24
|
+
else if (args.verb === undefined)
|
|
25
|
+
args.verb = a;
|
|
26
|
+
else
|
|
27
|
+
args.positional.push(a);
|
|
28
|
+
}
|
|
29
|
+
return args;
|
|
30
|
+
}
|
|
31
|
+
function fail(msg) {
|
|
32
|
+
process.stderr.write(`ctl-contention: ${msg}\n`);
|
|
33
|
+
process.exit(2);
|
|
34
|
+
}
|
|
35
|
+
async function main(argv) {
|
|
36
|
+
const args = parseCliArgs(argv);
|
|
37
|
+
const loaded = loadContentionConfig(args.config);
|
|
38
|
+
if (!loaded.ok)
|
|
39
|
+
fail(`invalid config: ${loaded.error}`);
|
|
40
|
+
const { config } = loaded;
|
|
41
|
+
switch (args.verb) {
|
|
42
|
+
case "run": {
|
|
43
|
+
if (args.command.length === 0)
|
|
44
|
+
fail("run needs a command after --");
|
|
45
|
+
const r = await runUnderContention({
|
|
46
|
+
command: args.command,
|
|
47
|
+
config,
|
|
48
|
+
outDir: args.out ?? "test-results/contention",
|
|
49
|
+
});
|
|
50
|
+
return r.exitCode;
|
|
51
|
+
}
|
|
52
|
+
case "wait-quiet": {
|
|
53
|
+
const r = await waitForQuiet({ rerun: config.rerun, intervalMs: config.intervalMs });
|
|
54
|
+
process.stderr.write(`ctl-contention: ${r.quiet ? "quiet" : "not quiet"} after ${Math.round(r.waitedMs / 1000)}s (${r.reason})\n`);
|
|
55
|
+
return 0;
|
|
56
|
+
}
|
|
57
|
+
case "config": {
|
|
58
|
+
const key = args.positional[0];
|
|
59
|
+
if (!key)
|
|
60
|
+
fail("config needs a key, e.g. rerun.max");
|
|
61
|
+
let v = config;
|
|
62
|
+
for (const part of key.split("."))
|
|
63
|
+
v = v?.[part];
|
|
64
|
+
if (v === undefined || (typeof v === "object" && v !== null))
|
|
65
|
+
fail(`no setting ${key}`);
|
|
66
|
+
process.stdout.write(`${String(v)}\n`);
|
|
67
|
+
return 0;
|
|
68
|
+
}
|
|
69
|
+
case "probe": {
|
|
70
|
+
const p = detectPressure();
|
|
71
|
+
const sample = p.available ? readSample(p, readFileText) : undefined;
|
|
72
|
+
process.stdout.write(`${JSON.stringify({ pressure: p, sample }, null, 2)}\n`);
|
|
73
|
+
return 0;
|
|
74
|
+
}
|
|
75
|
+
default:
|
|
76
|
+
fail(`unknown verb ${args.verb ?? "(none)"}: run | wait-quiet | config | probe`);
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
if (import.meta.main)
|
|
80
|
+
process.exit(await main(process.argv.slice(2)));
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
export { BunTestOutput, type Stream } from "./bun-output.js";
|
|
2
|
+
export { detectPressure, type FileRead, type FileReader, type PressureSource, type PsiLine, type PsiResource, parseLoadavg, parseMeminfo, parsePsi, readFileText, readSample, type Sample, } from "./psi.js";
|
|
3
|
+
export { hostContendedDuring, loadContentionConfig, type RunOptions, type RunResult, runUnderContention, TIMELINE_ENV, type WaitForQuietOptions, waitForQuiet, } from "./run.js";
|
|
4
|
+
export { ContentionSampler, type SamplerOptions } from "./sampler.js";
|
|
5
|
+
export { renderSummary } from "./summary.js";
|
|
6
|
+
export { parseTimeline, type Share, serialiseHeader, serialiseSample, type Timeline, type TimelineHeader, type Window, type WindowStats, type WindowStatsOptions, windowStats, } from "./timeline.js";
|
|
7
|
+
export { type AbortConfig, type AbortDecision, type AbortRecord, abortDecision, type ClassifiedFailure, type ClassifyConfig, type ClassifyInput, type ContentionAnswer, type ContentionConfig, ContentionConfigSchema, classifyRun, contendedDuring, DEFAULT_CONTENTION_CONFIG, exitCodeFor, type RerunConfig, type RunStatus, type RunVerdict, resolveContentionConfig, type SuiteFailure, type SuiteRun, UNCERTAIN_EXIT, } from "./verdict.js";
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
// Contention-aware integration verdicts: sample host pressure (Linux PSI) for
|
|
2
|
+
// the length of a suite, and tell a failure the host caused from one the
|
|
3
|
+
// product did. `ctl-contention` (./cli.ts) is the wrapper CI runs the suite
|
|
4
|
+
// under; a test can ask about its own window with hostContendedDuring.
|
|
5
|
+
export { BunTestOutput } from "./bun-output.js";
|
|
6
|
+
export { detectPressure, parseLoadavg, parseMeminfo, parsePsi, readFileText, readSample, } from "./psi.js";
|
|
7
|
+
export { hostContendedDuring, loadContentionConfig, runUnderContention, TIMELINE_ENV, waitForQuiet, } from "./run.js";
|
|
8
|
+
export { ContentionSampler } from "./sampler.js";
|
|
9
|
+
export { renderSummary } from "./summary.js";
|
|
10
|
+
export { parseTimeline, serialiseHeader, serialiseSample, windowStats, } from "./timeline.js";
|
|
11
|
+
export { abortDecision, ContentionConfigSchema, classifyRun, contendedDuring, DEFAULT_CONTENTION_CONFIG, exitCodeFor, resolveContentionConfig, UNCERTAIN_EXIT, } from "./verdict.js";
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
export interface PsiLine {
|
|
2
|
+
avg10: number;
|
|
3
|
+
avg60: number;
|
|
4
|
+
avg300: number;
|
|
5
|
+
/** Cumulative stall time, microseconds. */
|
|
6
|
+
total: number;
|
|
7
|
+
}
|
|
8
|
+
export interface PsiResource {
|
|
9
|
+
some: PsiLine;
|
|
10
|
+
/** Absent on kernels before 5.13 for cpu. */
|
|
11
|
+
full?: PsiLine;
|
|
12
|
+
}
|
|
13
|
+
export interface Sample {
|
|
14
|
+
/** Wall clock, ms since the epoch: the same clock tests stamp windows with. */
|
|
15
|
+
t: number;
|
|
16
|
+
cpu?: PsiResource;
|
|
17
|
+
memory?: PsiResource;
|
|
18
|
+
io?: PsiResource;
|
|
19
|
+
load?: [number, number, number];
|
|
20
|
+
memAvailableKb?: number;
|
|
21
|
+
memTotalKb?: number;
|
|
22
|
+
}
|
|
23
|
+
export type PressureSource = {
|
|
24
|
+
available: true;
|
|
25
|
+
source: "proc" | "cgroup";
|
|
26
|
+
cpuPath: string;
|
|
27
|
+
memoryPath: string;
|
|
28
|
+
ioPath?: string;
|
|
29
|
+
} | {
|
|
30
|
+
available: false;
|
|
31
|
+
reason: string;
|
|
32
|
+
};
|
|
33
|
+
export type FileRead = {
|
|
34
|
+
ok: true;
|
|
35
|
+
text: string;
|
|
36
|
+
} | {
|
|
37
|
+
ok: false;
|
|
38
|
+
error: string;
|
|
39
|
+
};
|
|
40
|
+
export type FileReader = (path: string) => FileRead;
|
|
41
|
+
export declare const readFileText: FileReader;
|
|
42
|
+
/** One PSI file. Averages come back as fractions (the kernel prints percent). */
|
|
43
|
+
export declare function parsePsi(text: string): PsiResource | undefined;
|
|
44
|
+
export declare function parseLoadavg(text: string): [number, number, number] | undefined;
|
|
45
|
+
export declare function parseMeminfo(text: string): {
|
|
46
|
+
memAvailableKb?: number;
|
|
47
|
+
memTotalKb?: number;
|
|
48
|
+
};
|
|
49
|
+
/** Where this process can read pressure from, or why it cannot. Both cpu and
|
|
50
|
+
* memory must read: a verdict on half the signal would be a guess. */
|
|
51
|
+
export declare function detectPressure(read?: FileReader): PressureSource;
|
|
52
|
+
export declare function readSample(source: Extract<PressureSource, {
|
|
53
|
+
available: true;
|
|
54
|
+
}>, read?: FileReader, nowMs?: number): Sample;
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
// Host pressure, read from Linux PSI (pressure stall information). `some` is
|
|
2
|
+
// the share of time at least one runnable task was stalled on the resource;
|
|
3
|
+
// `full` the share in which every non-idle task was. `total` is cumulative
|
|
4
|
+
// stall time in microseconds, which is what the window maths uses: the avg10
|
|
5
|
+
// the kernel prints lags a burst by up to ten seconds.
|
|
6
|
+
//
|
|
7
|
+
// Host-wide /proc/pressure is preferred over our own cgroup's files: the engine
|
|
8
|
+
// containers a suite launches are host siblings of the runner (docker outside
|
|
9
|
+
// of docker), so the runner's cgroup does not see the pressure they create.
|
|
10
|
+
import { readFileSync } from "node:fs";
|
|
11
|
+
export const readFileText = (path) => {
|
|
12
|
+
try {
|
|
13
|
+
return { ok: true, text: readFileSync(path, "utf8") };
|
|
14
|
+
}
|
|
15
|
+
catch (e) {
|
|
16
|
+
return { ok: false, error: e instanceof Error ? e.message : String(e) };
|
|
17
|
+
}
|
|
18
|
+
};
|
|
19
|
+
const PSI_LINE = /^(some|full) avg10=([\d.]+) avg60=([\d.]+) avg300=([\d.]+) total=(\d+)$/;
|
|
20
|
+
/** One PSI file. Averages come back as fractions (the kernel prints percent). */
|
|
21
|
+
export function parsePsi(text) {
|
|
22
|
+
const lines = {};
|
|
23
|
+
for (const raw of text.split("\n")) {
|
|
24
|
+
const m = PSI_LINE.exec(raw.trim());
|
|
25
|
+
if (!m)
|
|
26
|
+
continue;
|
|
27
|
+
lines[m[1]] = {
|
|
28
|
+
avg10: Number(m[2]) / 100,
|
|
29
|
+
avg60: Number(m[3]) / 100,
|
|
30
|
+
avg300: Number(m[4]) / 100,
|
|
31
|
+
total: Number(m[5]),
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
if (!lines.some)
|
|
35
|
+
return undefined;
|
|
36
|
+
return lines.full ? { some: lines.some, full: lines.full } : { some: lines.some };
|
|
37
|
+
}
|
|
38
|
+
export function parseLoadavg(text) {
|
|
39
|
+
const parts = text.trim().split(/\s+/).slice(0, 3).map(Number);
|
|
40
|
+
if (parts.length !== 3 || parts.some((n) => !Number.isFinite(n)))
|
|
41
|
+
return undefined;
|
|
42
|
+
return [parts[0], parts[1], parts[2]];
|
|
43
|
+
}
|
|
44
|
+
export function parseMeminfo(text) {
|
|
45
|
+
const out = {};
|
|
46
|
+
for (const line of text.split("\n")) {
|
|
47
|
+
const m = /^(MemAvailable|MemTotal):\s+(\d+) kB/.exec(line);
|
|
48
|
+
if (m?.[1] === "MemAvailable")
|
|
49
|
+
out.memAvailableKb = Number(m[2]);
|
|
50
|
+
if (m?.[1] === "MemTotal")
|
|
51
|
+
out.memTotalKb = Number(m[2]);
|
|
52
|
+
}
|
|
53
|
+
return out;
|
|
54
|
+
}
|
|
55
|
+
const CANDIDATES = [
|
|
56
|
+
{
|
|
57
|
+
source: "proc",
|
|
58
|
+
cpu: "/proc/pressure/cpu",
|
|
59
|
+
memory: "/proc/pressure/memory",
|
|
60
|
+
io: "/proc/pressure/io",
|
|
61
|
+
},
|
|
62
|
+
{
|
|
63
|
+
source: "cgroup",
|
|
64
|
+
cpu: "/sys/fs/cgroup/cpu.pressure",
|
|
65
|
+
memory: "/sys/fs/cgroup/memory.pressure",
|
|
66
|
+
io: "/sys/fs/cgroup/io.pressure",
|
|
67
|
+
},
|
|
68
|
+
];
|
|
69
|
+
function readPsi(read, path) {
|
|
70
|
+
const r = read(path);
|
|
71
|
+
if (!r.ok)
|
|
72
|
+
return { ok: false, why: `${path}: ${r.error}` };
|
|
73
|
+
const psi = parsePsi(r.text);
|
|
74
|
+
return psi ? { ok: true, psi } : { ok: false, why: `${path}: not PSI text` };
|
|
75
|
+
}
|
|
76
|
+
/** Where this process can read pressure from, or why it cannot. Both cpu and
|
|
77
|
+
* memory must read: a verdict on half the signal would be a guess. */
|
|
78
|
+
export function detectPressure(read = readFileText) {
|
|
79
|
+
const why = [];
|
|
80
|
+
for (const c of CANDIDATES) {
|
|
81
|
+
const cpu = readPsi(read, c.cpu);
|
|
82
|
+
const memory = readPsi(read, c.memory);
|
|
83
|
+
if (cpu.ok && memory.ok) {
|
|
84
|
+
const found = { available: true, source: c.source, cpuPath: c.cpu, memoryPath: c.memory };
|
|
85
|
+
return readPsi(read, c.io).ok ? { ...found, ioPath: c.io } : found;
|
|
86
|
+
}
|
|
87
|
+
if (!cpu.ok)
|
|
88
|
+
why.push(cpu.why);
|
|
89
|
+
if (!memory.ok)
|
|
90
|
+
why.push(memory.why);
|
|
91
|
+
}
|
|
92
|
+
return { available: false, reason: why.join("; ") };
|
|
93
|
+
}
|
|
94
|
+
export function readSample(source, read = readFileText, nowMs = Date.now()) {
|
|
95
|
+
const sample = { t: nowMs };
|
|
96
|
+
const psi = (path) => {
|
|
97
|
+
if (!path)
|
|
98
|
+
return undefined;
|
|
99
|
+
const r = readPsi(read, path);
|
|
100
|
+
return r.ok ? r.psi : undefined;
|
|
101
|
+
};
|
|
102
|
+
const cpu = psi(source.cpuPath);
|
|
103
|
+
const memory = psi(source.memoryPath);
|
|
104
|
+
const io = psi(source.ioPath);
|
|
105
|
+
if (cpu)
|
|
106
|
+
sample.cpu = cpu;
|
|
107
|
+
if (memory)
|
|
108
|
+
sample.memory = memory;
|
|
109
|
+
if (io)
|
|
110
|
+
sample.io = io;
|
|
111
|
+
const load = read("/proc/loadavg");
|
|
112
|
+
const parsedLoad = load.ok ? parseLoadavg(load.text) : undefined;
|
|
113
|
+
if (parsedLoad)
|
|
114
|
+
sample.load = parsedLoad;
|
|
115
|
+
const mem = read("/proc/meminfo");
|
|
116
|
+
if (mem.ok)
|
|
117
|
+
Object.assign(sample, parseMeminfo(mem.text));
|
|
118
|
+
return sample;
|
|
119
|
+
}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import { type PressureSource, type Sample } from "./psi.js";
|
|
2
|
+
import { type Window } from "./timeline.js";
|
|
3
|
+
import { type ClassifyConfig, type ContentionAnswer, type ContentionConfig, type RerunConfig, type RunVerdict } from "./verdict.js";
|
|
4
|
+
export declare const TIMELINE_ENV = "NORSK_CONTENTION_TIMELINE";
|
|
5
|
+
export interface RunOptions {
|
|
6
|
+
command: string[];
|
|
7
|
+
config: ContentionConfig;
|
|
8
|
+
outDir: string;
|
|
9
|
+
env?: Record<string, string | undefined>;
|
|
10
|
+
/** Detected from this host when absent. */
|
|
11
|
+
pressure?: PressureSource;
|
|
12
|
+
readSample?: (nowMs: number) => Sample;
|
|
13
|
+
stdout?: (chunk: Buffer) => void;
|
|
14
|
+
stderr?: (chunk: Buffer) => void;
|
|
15
|
+
log?: (line: string) => void;
|
|
16
|
+
}
|
|
17
|
+
export interface RunResult {
|
|
18
|
+
exitCode: number;
|
|
19
|
+
verdict: RunVerdict;
|
|
20
|
+
}
|
|
21
|
+
export declare function runUnderContention(opts: RunOptions): Promise<RunResult>;
|
|
22
|
+
/** The question a test asks about its own window, answered from the timeline
|
|
23
|
+
* the wrapper is writing. Outside the wrapper there is no timeline, and the
|
|
24
|
+
* answer says so rather than guessing. */
|
|
25
|
+
export declare function hostContendedDuring(w: Window, opts?: {
|
|
26
|
+
env?: Record<string, string | undefined>;
|
|
27
|
+
classify?: ClassifyConfig;
|
|
28
|
+
}): ContentionAnswer;
|
|
29
|
+
export interface WaitForQuietOptions {
|
|
30
|
+
rerun: RerunConfig;
|
|
31
|
+
intervalMs: number;
|
|
32
|
+
pressure?: PressureSource;
|
|
33
|
+
readSample?: (nowMs: number) => Sample;
|
|
34
|
+
now?: () => number;
|
|
35
|
+
sleep?: (ms: number) => Promise<void>;
|
|
36
|
+
}
|
|
37
|
+
/** Before a rerun: wait for a whole quiet window, or give up after maxWaitSec
|
|
38
|
+
* and let the rerun take its chances. */
|
|
39
|
+
export declare function waitForQuiet(opts: WaitForQuietOptions): Promise<{
|
|
40
|
+
quiet: boolean;
|
|
41
|
+
waitedMs: number;
|
|
42
|
+
reason: string;
|
|
43
|
+
}>;
|
|
44
|
+
export declare function loadContentionConfig(path: string | undefined): {
|
|
45
|
+
ok: true;
|
|
46
|
+
config: ContentionConfig;
|
|
47
|
+
} | {
|
|
48
|
+
ok: false;
|
|
49
|
+
error: string;
|
|
50
|
+
};
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
// runUnderContention: run a suite with the sampler alongside, watch its output
|
|
2
|
+
// for failing tests, stop it early if the host is starved past use, and turn
|
|
3
|
+
// the outcome into a verdict and an exit status. Writes, into outDir:
|
|
4
|
+
// pressure.jsonl the timeline (the suite reads it via NORSK_CONTENTION_TIMELINE)
|
|
5
|
+
// verdict.json the classification
|
|
6
|
+
// summary.md the same for a human, ready for $GITHUB_STEP_SUMMARY
|
|
7
|
+
//
|
|
8
|
+
// Output passes through byte for byte. Without a pressure signal the exit
|
|
9
|
+
// status is the suite's own, so a box without PSI behaves exactly as before.
|
|
10
|
+
import { spawn } from "node:child_process";
|
|
11
|
+
import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
12
|
+
import { constants } from "node:os";
|
|
13
|
+
import { join } from "node:path";
|
|
14
|
+
import { StringDecoder } from "node:string_decoder";
|
|
15
|
+
import { BunTestOutput } from "./bun-output.js";
|
|
16
|
+
import { detectPressure, readFileText, readSample } from "./psi.js";
|
|
17
|
+
import { ContentionSampler } from "./sampler.js";
|
|
18
|
+
import { renderSummary } from "./summary.js";
|
|
19
|
+
import { parseTimeline, windowStats } from "./timeline.js";
|
|
20
|
+
import { abortDecision, classifyRun, contendedDuring, DEFAULT_CONTENTION_CONFIG, exitCodeFor, pct, resolveContentionConfig, } from "./verdict.js";
|
|
21
|
+
export const TIMELINE_ENV = "NORSK_CONTENTION_TIMELINE";
|
|
22
|
+
/** After an abort's SIGTERM, how long the suite gets to exit before SIGKILL. */
|
|
23
|
+
const KILL_GRACE_MS = 15_000;
|
|
24
|
+
export async function runUnderContention(opts) {
|
|
25
|
+
const log = opts.log ?? ((l) => process.stderr.write(`ctl-contention: ${l}\n`));
|
|
26
|
+
const writeOut = opts.stdout ?? ((c) => process.stdout.write(c));
|
|
27
|
+
const writeErr = opts.stderr ?? ((c) => process.stderr.write(c));
|
|
28
|
+
const [cmd, ...args] = opts.command;
|
|
29
|
+
if (!cmd)
|
|
30
|
+
throw new Error("runUnderContention: empty command");
|
|
31
|
+
mkdirSync(opts.outDir, { recursive: true });
|
|
32
|
+
const timelinePath = join(opts.outDir, "pressure.jsonl");
|
|
33
|
+
const pressure = opts.pressure ?? detectPressure();
|
|
34
|
+
if (!pressure.available)
|
|
35
|
+
log(`pressure unavailable (${pressure.reason}); the suite's result will stand as it is`);
|
|
36
|
+
const tracker = new BunTestOutput();
|
|
37
|
+
const startMs = Date.now();
|
|
38
|
+
let aborted;
|
|
39
|
+
let externalSignal;
|
|
40
|
+
let child;
|
|
41
|
+
const killGroup = (sig) => {
|
|
42
|
+
if (child?.pid === undefined || child.exitCode !== null || child.signalCode !== null)
|
|
43
|
+
return;
|
|
44
|
+
try {
|
|
45
|
+
process.kill(-child.pid, sig);
|
|
46
|
+
}
|
|
47
|
+
catch {
|
|
48
|
+
child.kill(sig);
|
|
49
|
+
}
|
|
50
|
+
};
|
|
51
|
+
const sampler = new ContentionSampler({
|
|
52
|
+
pressure,
|
|
53
|
+
intervalMs: opts.config.intervalMs,
|
|
54
|
+
outPath: timelinePath,
|
|
55
|
+
...(opts.readSample ? { readSample: opts.readSample } : {}),
|
|
56
|
+
onSample: (s, smp) => {
|
|
57
|
+
if (aborted)
|
|
58
|
+
return;
|
|
59
|
+
const d = abortDecision(smp.samples, { startMs, nowMs: s.t }, opts.config.abort);
|
|
60
|
+
if (!d.abort)
|
|
61
|
+
return;
|
|
62
|
+
aborted = { rule: d.rule, reasons: d.reasons };
|
|
63
|
+
log(`stopping the suite, the host is too contended to judge it (${d.rule}): ${d.reasons.join("; ")}`);
|
|
64
|
+
killGroup("SIGTERM");
|
|
65
|
+
setTimeout(() => killGroup("SIGKILL"), KILL_GRACE_MS).unref();
|
|
66
|
+
},
|
|
67
|
+
});
|
|
68
|
+
sampler.start();
|
|
69
|
+
// Its own process group, so an abort or a forwarded signal reaches bun test
|
|
70
|
+
// and everything it started, not just the `bun run` in front of it.
|
|
71
|
+
child = spawn(cmd, args, {
|
|
72
|
+
env: { ...(opts.env ?? process.env), [TIMELINE_ENV]: timelinePath },
|
|
73
|
+
stdio: ["inherit", "pipe", "pipe"],
|
|
74
|
+
detached: true,
|
|
75
|
+
});
|
|
76
|
+
const pipe = (stream, src, write) => {
|
|
77
|
+
const decoder = new StringDecoder("utf8");
|
|
78
|
+
src?.on("data", (c) => {
|
|
79
|
+
write(c);
|
|
80
|
+
tracker.feed(decoder.write(c), stream);
|
|
81
|
+
});
|
|
82
|
+
};
|
|
83
|
+
pipe("stdout", child.stdout, writeOut);
|
|
84
|
+
pipe("stderr", child.stderr, writeErr);
|
|
85
|
+
const forward = (sig) => {
|
|
86
|
+
externalSignal = sig;
|
|
87
|
+
killGroup(sig);
|
|
88
|
+
};
|
|
89
|
+
const signals = ["SIGTERM", "SIGINT", "SIGHUP"];
|
|
90
|
+
for (const s of signals)
|
|
91
|
+
process.on(s, forward);
|
|
92
|
+
const spawned = child;
|
|
93
|
+
const { code, signal } = await new Promise((resolve) => {
|
|
94
|
+
spawned.on("error", (e) => {
|
|
95
|
+
log(`could not start ${cmd}: ${e.message}`);
|
|
96
|
+
resolve({ code: 127, signal: null });
|
|
97
|
+
});
|
|
98
|
+
spawned.on("close", (c, s) => resolve({ code: c, signal: s }));
|
|
99
|
+
});
|
|
100
|
+
for (const s of signals)
|
|
101
|
+
process.off(s, forward);
|
|
102
|
+
sampler.stop();
|
|
103
|
+
const endMs = Date.now();
|
|
104
|
+
const childCode = code ?? (signal ? 128 + (constants.signals[signal] ?? 0) : 1);
|
|
105
|
+
const timeline = sampler.timeline();
|
|
106
|
+
const verdict = classifyRun({
|
|
107
|
+
timeline,
|
|
108
|
+
failures: tracker.failures(),
|
|
109
|
+
run: {
|
|
110
|
+
startMs,
|
|
111
|
+
endMs,
|
|
112
|
+
exitCode: code,
|
|
113
|
+
signal: externalSignal ?? signal,
|
|
114
|
+
killedByUs: aborted !== undefined && externalSignal === undefined,
|
|
115
|
+
},
|
|
116
|
+
config: opts.config,
|
|
117
|
+
...(aborted ? { aborted } : {}),
|
|
118
|
+
});
|
|
119
|
+
writeFileSync(join(opts.outDir, "verdict.json"), `${JSON.stringify({ ...verdict, counts: tracker.counts() }, null, 2)}\n`);
|
|
120
|
+
writeFileSync(join(opts.outDir, "summary.md"), renderSummary(verdict, timeline));
|
|
121
|
+
log(`${verdict.status}: ${verdict.reason}`);
|
|
122
|
+
return { exitCode: exitCodeFor(verdict.status, childCode), verdict };
|
|
123
|
+
}
|
|
124
|
+
/** The question a test asks about its own window, answered from the timeline
|
|
125
|
+
* the wrapper is writing. Outside the wrapper there is no timeline, and the
|
|
126
|
+
* answer says so rather than guessing. */
|
|
127
|
+
export function hostContendedDuring(w, opts = {}) {
|
|
128
|
+
const path = (opts.env ?? process.env)[TIMELINE_ENV];
|
|
129
|
+
if (!path)
|
|
130
|
+
return { pressure: "unavailable", reason: `no contention timeline (${TIMELINE_ENV} is not set)` };
|
|
131
|
+
const read = readFileText(path);
|
|
132
|
+
if (!read.ok)
|
|
133
|
+
return { pressure: "unavailable", reason: `${path}: ${read.error}` };
|
|
134
|
+
const parsed = parseTimeline(read.text);
|
|
135
|
+
if (!parsed.ok)
|
|
136
|
+
return { pressure: "unavailable", reason: `${path}: ${parsed.error}` };
|
|
137
|
+
return contendedDuring(parsed.timeline, w, opts.classify ?? DEFAULT_CONTENTION_CONFIG.classify);
|
|
138
|
+
}
|
|
139
|
+
/** Before a rerun: wait for a whole quiet window, or give up after maxWaitSec
|
|
140
|
+
* and let the rerun take its chances. */
|
|
141
|
+
export async function waitForQuiet(opts) {
|
|
142
|
+
const pressure = opts.pressure ?? detectPressure();
|
|
143
|
+
if (!pressure.available)
|
|
144
|
+
return { quiet: true, waitedMs: 0, reason: `pressure unavailable: ${pressure.reason}` };
|
|
145
|
+
const now = opts.now ?? Date.now;
|
|
146
|
+
const sleep = opts.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
|
|
147
|
+
const read = opts.readSample ?? ((t) => readSample(pressure, readFileText, t));
|
|
148
|
+
const { rerun } = opts;
|
|
149
|
+
const started = now();
|
|
150
|
+
const samples = [read(started)];
|
|
151
|
+
for (;;) {
|
|
152
|
+
await sleep(opts.intervalMs);
|
|
153
|
+
const t = now();
|
|
154
|
+
samples.push(read(t));
|
|
155
|
+
const waitedMs = t - started;
|
|
156
|
+
if (waitedMs >= rerun.quietWindowSec * 1000) {
|
|
157
|
+
const s = windowStats(samples, { fromMs: t - rerun.quietWindowSec * 1000, toMs: t }, {
|
|
158
|
+
peakWindowSec: rerun.quietWindowSec,
|
|
159
|
+
});
|
|
160
|
+
if (s.coverage >= 0.8 && s.cpuSome.mean < rerun.quietCpuSome && s.memSome.mean < rerun.quietMemSome) {
|
|
161
|
+
return {
|
|
162
|
+
quiet: true,
|
|
163
|
+
waitedMs,
|
|
164
|
+
reason: `cpu some ${pct(s.cpuSome.mean)} over the last ${rerun.quietWindowSec}s`,
|
|
165
|
+
};
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
if (waitedMs >= rerun.maxWaitSec * 1000) {
|
|
169
|
+
return { quiet: false, waitedMs, reason: `still contended after ${rerun.maxWaitSec}s` };
|
|
170
|
+
}
|
|
171
|
+
if (samples.length > (4 * (rerun.quietWindowSec * 1000)) / opts.intervalMs)
|
|
172
|
+
samples.shift();
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
export function loadContentionConfig(path) {
|
|
176
|
+
if (!path)
|
|
177
|
+
return { ok: true, config: DEFAULT_CONTENTION_CONFIG };
|
|
178
|
+
let raw;
|
|
179
|
+
try {
|
|
180
|
+
raw = JSON.parse(readFileSync(path, "utf8"));
|
|
181
|
+
}
|
|
182
|
+
catch (e) {
|
|
183
|
+
return { ok: false, error: `${path}: ${e instanceof Error ? e.message : String(e)}` };
|
|
184
|
+
}
|
|
185
|
+
const r = resolveContentionConfig(raw);
|
|
186
|
+
return r.ok ? r : { ok: false, error: `${path}: ${r.error}` };
|
|
187
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { type PressureSource, type Sample } from "./psi.js";
|
|
2
|
+
import { type Timeline } from "./timeline.js";
|
|
3
|
+
export interface SamplerOptions {
|
|
4
|
+
pressure: PressureSource;
|
|
5
|
+
intervalMs: number;
|
|
6
|
+
outPath?: string;
|
|
7
|
+
readSample?: (nowMs: number) => Sample;
|
|
8
|
+
now?: () => number;
|
|
9
|
+
onSample?: (sample: Sample, sampler: ContentionSampler) => void;
|
|
10
|
+
}
|
|
11
|
+
export declare class ContentionSampler {
|
|
12
|
+
private readonly opts;
|
|
13
|
+
readonly samples: Sample[];
|
|
14
|
+
private header;
|
|
15
|
+
private timer;
|
|
16
|
+
private readonly now;
|
|
17
|
+
private readonly read;
|
|
18
|
+
constructor(opts: SamplerOptions);
|
|
19
|
+
start(): void;
|
|
20
|
+
stop(): void;
|
|
21
|
+
timeline(): Timeline;
|
|
22
|
+
private tick;
|
|
23
|
+
}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
// Samples host pressure once a tick for as long as a suite runs, appending each
|
|
2
|
+
// sample to the timeline file as it is taken. Without a pressure signal it
|
|
3
|
+
// writes the header (which says why) and samples nothing.
|
|
4
|
+
import { appendFileSync, mkdirSync, writeFileSync } from "node:fs";
|
|
5
|
+
import { cpus, hostname } from "node:os";
|
|
6
|
+
import { dirname } from "node:path";
|
|
7
|
+
import { readFileText, readSample } from "./psi.js";
|
|
8
|
+
import { serialiseHeader, serialiseSample } from "./timeline.js";
|
|
9
|
+
export class ContentionSampler {
|
|
10
|
+
opts;
|
|
11
|
+
samples = [];
|
|
12
|
+
header;
|
|
13
|
+
timer;
|
|
14
|
+
now;
|
|
15
|
+
read;
|
|
16
|
+
constructor(opts) {
|
|
17
|
+
this.opts = opts;
|
|
18
|
+
this.now = opts.now ?? Date.now;
|
|
19
|
+
const p = opts.pressure;
|
|
20
|
+
this.read = opts.readSample ?? (p.available ? (t) => readSample(p, readFileText, t) : undefined);
|
|
21
|
+
}
|
|
22
|
+
start() {
|
|
23
|
+
this.header = {
|
|
24
|
+
kind: "header",
|
|
25
|
+
version: 1,
|
|
26
|
+
startedAt: this.now(),
|
|
27
|
+
intervalMs: this.opts.intervalMs,
|
|
28
|
+
host: hostname(),
|
|
29
|
+
ncpu: cpus().length,
|
|
30
|
+
pressure: this.opts.pressure,
|
|
31
|
+
};
|
|
32
|
+
if (this.opts.outPath) {
|
|
33
|
+
mkdirSync(dirname(this.opts.outPath), { recursive: true });
|
|
34
|
+
writeFileSync(this.opts.outPath, serialiseHeader(this.header));
|
|
35
|
+
}
|
|
36
|
+
if (!this.opts.pressure.available)
|
|
37
|
+
return;
|
|
38
|
+
this.tick();
|
|
39
|
+
this.timer = setInterval(() => this.tick(), this.opts.intervalMs);
|
|
40
|
+
}
|
|
41
|
+
stop() {
|
|
42
|
+
if (this.timer === undefined)
|
|
43
|
+
return;
|
|
44
|
+
clearInterval(this.timer);
|
|
45
|
+
this.timer = undefined;
|
|
46
|
+
this.tick();
|
|
47
|
+
}
|
|
48
|
+
timeline() {
|
|
49
|
+
if (!this.header)
|
|
50
|
+
throw new Error("ContentionSampler.timeline() before start()");
|
|
51
|
+
return { header: this.header, samples: this.samples };
|
|
52
|
+
}
|
|
53
|
+
tick() {
|
|
54
|
+
if (!this.read)
|
|
55
|
+
return;
|
|
56
|
+
const s = this.read(this.now());
|
|
57
|
+
this.samples.push(s);
|
|
58
|
+
if (this.opts.outPath)
|
|
59
|
+
appendFileSync(this.opts.outPath, serialiseSample(s));
|
|
60
|
+
this.opts.onSample?.(s, this);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import { windowStats } from "./timeline.js";
|
|
2
|
+
import { pct } from "./verdict.js";
|
|
3
|
+
const MAX_MINUTE_ROWS = 120;
|
|
4
|
+
const cell = (s) => s.replace(/\|/g, "\\|").replace(/\n/g, " ");
|
|
5
|
+
function offset(ms) {
|
|
6
|
+
const s = Math.max(0, Math.round(ms / 1000));
|
|
7
|
+
return `+${Math.floor(s / 60)}m${String(s % 60).padStart(2, "0")}s`;
|
|
8
|
+
}
|
|
9
|
+
function answerCells(a) {
|
|
10
|
+
if (a.pressure === "unavailable")
|
|
11
|
+
return ["n/a", "n/a", "n/a", "not judged"];
|
|
12
|
+
const s = a.stats;
|
|
13
|
+
return [
|
|
14
|
+
`${pct(s.cpuSome.mean)} / ${pct(s.cpuSome.peak)}`,
|
|
15
|
+
pct(s.memSome.mean),
|
|
16
|
+
pct(s.memFull.peak),
|
|
17
|
+
a.contended ? `contended: ${a.reasons.join("; ")}` : `quiet${a.reasons.length ? `: ${a.reasons.join("; ")}` : ""}`,
|
|
18
|
+
];
|
|
19
|
+
}
|
|
20
|
+
export function renderSummary(v, t) {
|
|
21
|
+
const h = t.header;
|
|
22
|
+
const out = [`### Integration host contention: **${v.status}**`, "", cell(v.reason), ""];
|
|
23
|
+
if (!h.pressure.available) {
|
|
24
|
+
out.push(`Host \`${h.host}\`: pressure unavailable (${cell(h.pressure.reason)}). No verdict was changed; the result is the suite's own.`);
|
|
25
|
+
return `${out.join("\n")}\n`;
|
|
26
|
+
}
|
|
27
|
+
const minutes = Math.ceil((v.endMs - v.startMs) / 60_000);
|
|
28
|
+
out.push(`Host \`${h.host}\`, ${h.ncpu} CPUs; pressure from \`${h.pressure.source === "proc" ? "/proc/pressure (host-wide)" : "/sys/fs/cgroup (this cgroup)"}\`, ` +
|
|
29
|
+
`sampled every ${h.intervalMs} ms, ${t.samples.length} samples over ${minutes} min.`);
|
|
30
|
+
if (v.failures.length) {
|
|
31
|
+
out.push("", "#### Failures", "", "| Test | Window | CPU some mean / peak | Memory some mean | Memory full peak | Classified |", "| --- | --- | --- | --- | --- | --- |");
|
|
32
|
+
for (const f of v.failures) {
|
|
33
|
+
const where = `${offset(f.fromMs - v.startMs)} for ${Math.round((f.toMs - f.fromMs) / 1000)}s`;
|
|
34
|
+
out.push(`| ${[f.name, where, ...answerCells(f.answer)].map(cell).join(" | ")} |`);
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
out.push("", "#### Pressure, per minute", "", "| Minute | CPU some (peak 10s) | Memory some | Memory full | Load per CPU | Memory available |", "| --- | --- | --- | --- | --- | --- |");
|
|
38
|
+
for (let m = 0; m < Math.min(minutes, MAX_MINUTE_ROWS); m++) {
|
|
39
|
+
const fromMs = v.startMs + m * 60_000;
|
|
40
|
+
const s = windowStats(t.samples, { fromMs, toMs: Math.min(fromMs + 60_000, v.endMs) }, { peakWindowSec: 10, ncpu: h.ncpu });
|
|
41
|
+
const opt = (x, f) => (x === undefined ? "n/a" : f(x));
|
|
42
|
+
out.push(`| +${m}m | ${pct(s.cpuSome.mean)} (${pct(s.cpuSome.peak)}) | ${pct(s.memSome.mean)} | ${pct(s.memFull.mean)} | ` +
|
|
43
|
+
`${opt(s.loadPerCpuMax, (n) => n.toFixed(2))} | ${opt(s.memAvailableMinFraction, pct)} |`);
|
|
44
|
+
}
|
|
45
|
+
if (minutes > MAX_MINUTE_ROWS)
|
|
46
|
+
out.push("", `(first ${MAX_MINUTE_ROWS} minutes shown; the full timeline is in the artifact)`);
|
|
47
|
+
return `${out.join("\n")}\n`;
|
|
48
|
+
}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import type { PressureSource, Sample } from "./psi.js";
|
|
2
|
+
export interface TimelineHeader {
|
|
3
|
+
kind: "header";
|
|
4
|
+
version: 1;
|
|
5
|
+
startedAt: number;
|
|
6
|
+
intervalMs: number;
|
|
7
|
+
host: string;
|
|
8
|
+
ncpu: number;
|
|
9
|
+
pressure: PressureSource;
|
|
10
|
+
}
|
|
11
|
+
export interface Timeline {
|
|
12
|
+
header: TimelineHeader;
|
|
13
|
+
samples: Sample[];
|
|
14
|
+
}
|
|
15
|
+
export declare function serialiseHeader(h: TimelineHeader): string;
|
|
16
|
+
export declare function serialiseSample(s: Sample): string;
|
|
17
|
+
export declare function parseTimeline(text: string): {
|
|
18
|
+
ok: true;
|
|
19
|
+
timeline: Timeline;
|
|
20
|
+
} | {
|
|
21
|
+
ok: false;
|
|
22
|
+
error: string;
|
|
23
|
+
};
|
|
24
|
+
export interface Window {
|
|
25
|
+
fromMs: number;
|
|
26
|
+
toMs: number;
|
|
27
|
+
}
|
|
28
|
+
export interface Share {
|
|
29
|
+
mean: number;
|
|
30
|
+
/** The worst mean over any `peakWindowSec` stretch inside the window. */
|
|
31
|
+
peak: number;
|
|
32
|
+
}
|
|
33
|
+
export interface WindowStats extends Window {
|
|
34
|
+
samples: number;
|
|
35
|
+
/** Share of the window covered by sampler ticks no further apart than maxGapMs. */
|
|
36
|
+
coverage: number;
|
|
37
|
+
cpuSome: Share;
|
|
38
|
+
memSome: Share;
|
|
39
|
+
memFull: Share;
|
|
40
|
+
loadPerCpuMax?: number;
|
|
41
|
+
memAvailableMinFraction?: number;
|
|
42
|
+
}
|
|
43
|
+
export interface WindowStatsOptions {
|
|
44
|
+
peakWindowSec: number;
|
|
45
|
+
ncpu?: number;
|
|
46
|
+
/** A gap longer than this between two ticks is not counted as observed: a
|
|
47
|
+
* sampler that went quiet for half a minute did not see that half minute. */
|
|
48
|
+
maxGapMs?: number;
|
|
49
|
+
}
|
|
50
|
+
export declare function windowStats(samples: readonly Sample[], w: Window, opts: WindowStatsOptions): WindowStats;
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
export function serialiseHeader(h) {
|
|
2
|
+
return `${JSON.stringify(h)}\n`;
|
|
3
|
+
}
|
|
4
|
+
export function serialiseSample(s) {
|
|
5
|
+
return `${JSON.stringify({ kind: "sample", ...s })}\n`;
|
|
6
|
+
}
|
|
7
|
+
export function parseTimeline(text) {
|
|
8
|
+
let header;
|
|
9
|
+
const samples = [];
|
|
10
|
+
for (const line of text.split("\n")) {
|
|
11
|
+
if (!line.trim())
|
|
12
|
+
continue;
|
|
13
|
+
let row;
|
|
14
|
+
try {
|
|
15
|
+
row = JSON.parse(line);
|
|
16
|
+
}
|
|
17
|
+
catch {
|
|
18
|
+
continue;
|
|
19
|
+
}
|
|
20
|
+
if (typeof row !== "object" || row === null)
|
|
21
|
+
continue;
|
|
22
|
+
const { kind, ...rest } = row;
|
|
23
|
+
if (kind === "header")
|
|
24
|
+
header = row;
|
|
25
|
+
else if (kind === "sample" && typeof rest.t === "number")
|
|
26
|
+
samples.push(rest);
|
|
27
|
+
}
|
|
28
|
+
if (!header)
|
|
29
|
+
return { ok: false, error: "the timeline has no header line" };
|
|
30
|
+
return { ok: true, timeline: { header, samples } };
|
|
31
|
+
}
|
|
32
|
+
const METRICS = {
|
|
33
|
+
cpuSome: (s) => s.cpu?.some.total,
|
|
34
|
+
memSome: (s) => s.memory?.some.total,
|
|
35
|
+
memFull: (s) => s.memory?.full?.total,
|
|
36
|
+
};
|
|
37
|
+
function pieces(samples, w, metric, maxGapMs) {
|
|
38
|
+
const out = [];
|
|
39
|
+
for (let i = 1; i < samples.length; i++) {
|
|
40
|
+
const a = samples[i - 1];
|
|
41
|
+
const b = samples[i];
|
|
42
|
+
const dt = b.t - a.t;
|
|
43
|
+
if (dt <= 0 || dt > maxGapMs)
|
|
44
|
+
continue;
|
|
45
|
+
const ms = Math.min(b.t, w.toMs) - Math.max(a.t, w.fromMs);
|
|
46
|
+
if (ms <= 0)
|
|
47
|
+
continue;
|
|
48
|
+
const va = metric(a);
|
|
49
|
+
const vb = metric(b);
|
|
50
|
+
if (va === undefined || vb === undefined || vb < va)
|
|
51
|
+
continue;
|
|
52
|
+
out.push({ ms, share: Math.min(1, Math.max(0, (vb - va) / (dt * 1000))) });
|
|
53
|
+
}
|
|
54
|
+
return out;
|
|
55
|
+
}
|
|
56
|
+
function share(ps, peakMs) {
|
|
57
|
+
const totalMs = ps.reduce((n, p) => n + p.ms, 0);
|
|
58
|
+
if (totalMs === 0)
|
|
59
|
+
return { mean: 0, peak: 0 };
|
|
60
|
+
const mean = ps.reduce((n, p) => n + p.share * p.ms, 0) / totalMs;
|
|
61
|
+
if (totalMs <= peakMs)
|
|
62
|
+
return { mean, peak: mean };
|
|
63
|
+
let peak = 0;
|
|
64
|
+
for (let j = 0; j < ps.length; j++) {
|
|
65
|
+
let ms = 0;
|
|
66
|
+
let stalled = 0;
|
|
67
|
+
for (let i = j; i >= 0 && ms < peakMs; i--) {
|
|
68
|
+
const p = ps[i];
|
|
69
|
+
ms += p.ms;
|
|
70
|
+
stalled += p.share * p.ms;
|
|
71
|
+
}
|
|
72
|
+
if (ms >= peakMs)
|
|
73
|
+
peak = Math.max(peak, stalled / ms);
|
|
74
|
+
}
|
|
75
|
+
return { mean, peak };
|
|
76
|
+
}
|
|
77
|
+
export function windowStats(samples, w, opts) {
|
|
78
|
+
const maxGapMs = opts.maxGapMs ?? 5000;
|
|
79
|
+
const peakMs = opts.peakWindowSec * 1000;
|
|
80
|
+
const lengthMs = w.toMs - w.fromMs;
|
|
81
|
+
const observed = pieces(samples, w, (s) => s.t, maxGapMs).reduce((n, p) => n + p.ms, 0);
|
|
82
|
+
const inside = samples.filter((s) => s.t >= w.fromMs && s.t <= w.toMs);
|
|
83
|
+
const stats = {
|
|
84
|
+
...w,
|
|
85
|
+
samples: inside.length,
|
|
86
|
+
coverage: lengthMs > 0 ? Math.min(1, observed / lengthMs) : 0,
|
|
87
|
+
cpuSome: share(pieces(samples, w, METRICS.cpuSome, maxGapMs), peakMs),
|
|
88
|
+
memSome: share(pieces(samples, w, METRICS.memSome, maxGapMs), peakMs),
|
|
89
|
+
memFull: share(pieces(samples, w, METRICS.memFull, maxGapMs), peakMs),
|
|
90
|
+
};
|
|
91
|
+
const ncpu = opts.ncpu;
|
|
92
|
+
const loads = inside.flatMap((s) => (s.load && ncpu ? [s.load[0] / ncpu] : []));
|
|
93
|
+
if (loads.length)
|
|
94
|
+
stats.loadPerCpuMax = Math.max(...loads);
|
|
95
|
+
const mem = inside.flatMap((s) => (s.memAvailableKb && s.memTotalKb ? [s.memAvailableKb / s.memTotalKb] : []));
|
|
96
|
+
if (mem.length)
|
|
97
|
+
stats.memAvailableMinFraction = Math.min(...mem);
|
|
98
|
+
return stats;
|
|
99
|
+
}
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import type { Timeline, Window, WindowStats } from "./timeline.js";
|
|
3
|
+
/** EX_TEMPFAIL: "try again later". Distinct from a test failure (1), a crash,
|
|
4
|
+
* and a signal death (128+n). */
|
|
5
|
+
export declare const UNCERTAIN_EXIT = 75;
|
|
6
|
+
export declare const ContentionConfigSchema: z.ZodObject<{
|
|
7
|
+
intervalMs: z.ZodDefault<z.ZodNumber>;
|
|
8
|
+
classify: z.ZodPrefault<z.ZodObject<{
|
|
9
|
+
cpuSomeMean: z.ZodDefault<z.ZodNumber>;
|
|
10
|
+
cpuSomePeak: z.ZodDefault<z.ZodNumber>;
|
|
11
|
+
memSomeMean: z.ZodDefault<z.ZodNumber>;
|
|
12
|
+
memFullPeak: z.ZodDefault<z.ZodNumber>;
|
|
13
|
+
peakWindowSec: z.ZodDefault<z.ZodNumber>;
|
|
14
|
+
marginSec: z.ZodDefault<z.ZodNumber>;
|
|
15
|
+
minCoverage: z.ZodDefault<z.ZodNumber>;
|
|
16
|
+
killWindowSec: z.ZodDefault<z.ZodNumber>;
|
|
17
|
+
}, z.core.$strict>>;
|
|
18
|
+
abort: z.ZodPrefault<z.ZodObject<{
|
|
19
|
+
enabled: z.ZodDefault<z.ZodBoolean>;
|
|
20
|
+
cpuSome: z.ZodDefault<z.ZodNumber>;
|
|
21
|
+
memSome: z.ZodDefault<z.ZodNumber>;
|
|
22
|
+
memFull: z.ZodDefault<z.ZodNumber>;
|
|
23
|
+
earlyWindowSec: z.ZodDefault<z.ZodNumber>;
|
|
24
|
+
earlyWithinSec: z.ZodDefault<z.ZodNumber>;
|
|
25
|
+
sustainedWindowSec: z.ZodDefault<z.ZodNumber>;
|
|
26
|
+
minCoverage: z.ZodDefault<z.ZodNumber>;
|
|
27
|
+
}, z.core.$strict>>;
|
|
28
|
+
rerun: z.ZodPrefault<z.ZodObject<{
|
|
29
|
+
max: z.ZodDefault<z.ZodNumber>;
|
|
30
|
+
quietCpuSome: z.ZodDefault<z.ZodNumber>;
|
|
31
|
+
quietMemSome: z.ZodDefault<z.ZodNumber>;
|
|
32
|
+
quietWindowSec: z.ZodDefault<z.ZodNumber>;
|
|
33
|
+
maxWaitSec: z.ZodDefault<z.ZodNumber>;
|
|
34
|
+
}, z.core.$strict>>;
|
|
35
|
+
}, z.core.$strict>;
|
|
36
|
+
export type ContentionConfig = z.infer<typeof ContentionConfigSchema>;
|
|
37
|
+
export type ClassifyConfig = ContentionConfig["classify"];
|
|
38
|
+
export type AbortConfig = ContentionConfig["abort"];
|
|
39
|
+
export type RerunConfig = ContentionConfig["rerun"];
|
|
40
|
+
export declare const DEFAULT_CONTENTION_CONFIG: ContentionConfig;
|
|
41
|
+
export declare function resolveContentionConfig(raw: unknown): {
|
|
42
|
+
ok: true;
|
|
43
|
+
config: ContentionConfig;
|
|
44
|
+
} | {
|
|
45
|
+
ok: false;
|
|
46
|
+
error: string;
|
|
47
|
+
};
|
|
48
|
+
export type ContentionAnswer = {
|
|
49
|
+
pressure: "unavailable";
|
|
50
|
+
reason: string;
|
|
51
|
+
} | {
|
|
52
|
+
pressure: "available";
|
|
53
|
+
contended: boolean;
|
|
54
|
+
reasons: string[];
|
|
55
|
+
stats: WindowStats;
|
|
56
|
+
};
|
|
57
|
+
export declare const pct: (x: number) => string;
|
|
58
|
+
export declare function contendedDuring(t: Timeline, w: Window, th: ClassifyConfig): ContentionAnswer;
|
|
59
|
+
export interface SuiteFailure {
|
|
60
|
+
name: string;
|
|
61
|
+
file?: string;
|
|
62
|
+
/** When its failure line arrived. */
|
|
63
|
+
endMs: number;
|
|
64
|
+
durationMs: number;
|
|
65
|
+
}
|
|
66
|
+
export interface SuiteRun {
|
|
67
|
+
startMs: number;
|
|
68
|
+
endMs: number;
|
|
69
|
+
exitCode: number | null;
|
|
70
|
+
signal: string | null;
|
|
71
|
+
/** True when the wrapper stopped the suite itself (the early abort). */
|
|
72
|
+
killedByUs: boolean;
|
|
73
|
+
}
|
|
74
|
+
export interface AbortRecord {
|
|
75
|
+
rule: "early" | "sustained";
|
|
76
|
+
reasons: string[];
|
|
77
|
+
}
|
|
78
|
+
export type RunStatus = "pass" | "fail" | "uncertain";
|
|
79
|
+
export interface ClassifiedFailure {
|
|
80
|
+
name: string;
|
|
81
|
+
file?: string;
|
|
82
|
+
fromMs: number;
|
|
83
|
+
toMs: number;
|
|
84
|
+
contended: boolean;
|
|
85
|
+
answer: ContentionAnswer;
|
|
86
|
+
}
|
|
87
|
+
export interface RunVerdict {
|
|
88
|
+
status: RunStatus;
|
|
89
|
+
reason: string;
|
|
90
|
+
pressure: "available" | "unavailable";
|
|
91
|
+
pressureReason?: string;
|
|
92
|
+
startMs: number;
|
|
93
|
+
endMs: number;
|
|
94
|
+
exitCode: number | null;
|
|
95
|
+
signal: string | null;
|
|
96
|
+
aborted?: AbortRecord;
|
|
97
|
+
failures: ClassifiedFailure[];
|
|
98
|
+
}
|
|
99
|
+
export interface ClassifyInput {
|
|
100
|
+
timeline: Timeline;
|
|
101
|
+
failures: readonly SuiteFailure[];
|
|
102
|
+
run: SuiteRun;
|
|
103
|
+
config: ContentionConfig;
|
|
104
|
+
aborted?: AbortRecord;
|
|
105
|
+
}
|
|
106
|
+
export declare function classifyRun(input: ClassifyInput): RunVerdict;
|
|
107
|
+
export type AbortDecision = {
|
|
108
|
+
abort: false;
|
|
109
|
+
} | ({
|
|
110
|
+
abort: true;
|
|
111
|
+
stats: WindowStats;
|
|
112
|
+
} & AbortRecord);
|
|
113
|
+
export declare function abortDecision(samples: Timeline["samples"], at: {
|
|
114
|
+
startMs: number;
|
|
115
|
+
nowMs: number;
|
|
116
|
+
}, cfg: AbortConfig): AbortDecision;
|
|
117
|
+
export declare function exitCodeFor(status: RunStatus, childCode: number | null): number;
|
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
// Contention verdicts. One question, "was the host contended over this
|
|
2
|
+
// window?", and three uses of it: a test asking about its own window, the
|
|
3
|
+
// classification of a finished suite, and the early stop of one that cannot
|
|
4
|
+
// produce a result worth reading.
|
|
5
|
+
//
|
|
6
|
+
// The rules every verdict keeps: a failure on a quiet box stays a failure, a
|
|
7
|
+
// pass stays a pass however busy the box was, and with no pressure signal the
|
|
8
|
+
// suite's own result stands untouched.
|
|
9
|
+
//
|
|
10
|
+
// Default thresholds, and why. PSI `some` for cpu is the share of time at
|
|
11
|
+
// least one runnable task waited for a cpu, so it bounds from above the share
|
|
12
|
+
// any one thread (a browser main thread, an engine scheduler) spent waiting.
|
|
13
|
+
// commentary's own browser control saw passing runs starved for at most ~5%
|
|
14
|
+
// of a window and failing ones for 6-80%, with the engine up to 50 s behind
|
|
15
|
+
// realtime and a 0.55 s switch taking 5-20 s. A mean cpu stall of 25% is five
|
|
16
|
+
// times the passing bar; 60% over any 10 s catches a stall long enough to
|
|
17
|
+
// blow a seconds-scale timing assertion that a long test's mean would dilute.
|
|
18
|
+
// Memory stalls are rarer and worse: 10% `some` means reclaim is a constant
|
|
19
|
+
// cost, and 5% `full` over 10 s is half a second in which nothing ran at all.
|
|
20
|
+
// Calling a real failure uncertain costs a rerun; calling a starved one red
|
|
21
|
+
// costs a human an investigation of nothing. The bars sit where the first is
|
|
22
|
+
// unlikely, and are config, so test2's recorded timelines can move them.
|
|
23
|
+
import { z } from "zod";
|
|
24
|
+
import { windowStats } from "./timeline.js";
|
|
25
|
+
/** EX_TEMPFAIL: "try again later". Distinct from a test failure (1), a crash,
|
|
26
|
+
* and a signal death (128+n). */
|
|
27
|
+
export const UNCERTAIN_EXIT = 75;
|
|
28
|
+
const share = z.number().min(0).max(1);
|
|
29
|
+
const seconds = z.number().positive();
|
|
30
|
+
const ClassifySchema = z.strictObject({
|
|
31
|
+
cpuSomeMean: share.default(0.25),
|
|
32
|
+
cpuSomePeak: share.default(0.6),
|
|
33
|
+
memSomeMean: share.default(0.1),
|
|
34
|
+
memFullPeak: share.default(0.05),
|
|
35
|
+
peakWindowSec: seconds.default(10),
|
|
36
|
+
/** Judged on the window widened by this much each side, clipped to the run:
|
|
37
|
+
* the stall that broke a test can land just before its failure line. */
|
|
38
|
+
marginSec: z.number().min(0).default(5),
|
|
39
|
+
/** Below this share sampled, the window is not judged at all. */
|
|
40
|
+
minCoverage: share.default(0.5),
|
|
41
|
+
/** A suite killed from outside is judged on this much of its end. */
|
|
42
|
+
killWindowSec: seconds.default(120),
|
|
43
|
+
});
|
|
44
|
+
const AbortSchema = z.strictObject({
|
|
45
|
+
enabled: z.boolean().default(true),
|
|
46
|
+
cpuSome: share.default(0.7),
|
|
47
|
+
memSome: share.default(0.3),
|
|
48
|
+
memFull: share.default(0.1),
|
|
49
|
+
/** Starved for this long in the first earlyWithinSec: nothing it runs will mean anything. */
|
|
50
|
+
earlyWindowSec: seconds.default(60),
|
|
51
|
+
earlyWithinSec: seconds.default(300),
|
|
52
|
+
/** Starved for this long anywhere: stop rather than run on for half an hour. */
|
|
53
|
+
sustainedWindowSec: seconds.default(300),
|
|
54
|
+
minCoverage: share.default(0.8),
|
|
55
|
+
});
|
|
56
|
+
const RerunSchema = z.strictObject({
|
|
57
|
+
/** Reruns in the same job after an uncertain attempt. 0 turns them off. */
|
|
58
|
+
max: z.number().int().min(0).max(3).default(1),
|
|
59
|
+
quietCpuSome: share.default(0.1),
|
|
60
|
+
quietMemSome: share.default(0.05),
|
|
61
|
+
quietWindowSec: seconds.default(30),
|
|
62
|
+
/** How long to wait for the host to go quiet before rerunning anyway. */
|
|
63
|
+
maxWaitSec: z.number().min(0).default(600),
|
|
64
|
+
});
|
|
65
|
+
export const ContentionConfigSchema = z.strictObject({
|
|
66
|
+
intervalMs: z.number().int().min(10).default(1000),
|
|
67
|
+
classify: ClassifySchema.prefault({}),
|
|
68
|
+
abort: AbortSchema.prefault({}),
|
|
69
|
+
rerun: RerunSchema.prefault({}),
|
|
70
|
+
});
|
|
71
|
+
export const DEFAULT_CONTENTION_CONFIG = ContentionConfigSchema.parse({});
|
|
72
|
+
export function resolveContentionConfig(raw) {
|
|
73
|
+
const r = ContentionConfigSchema.safeParse(raw);
|
|
74
|
+
return r.success ? { ok: true, config: r.data } : { ok: false, error: z.prettifyError(r.error) };
|
|
75
|
+
}
|
|
76
|
+
export const pct = (x) => `${(x * 100).toFixed(1)}%`;
|
|
77
|
+
function maxGapMs(t) {
|
|
78
|
+
return Math.max(5000, 5 * t.header.intervalMs);
|
|
79
|
+
}
|
|
80
|
+
export function contendedDuring(t, w, th) {
|
|
81
|
+
if (!t.header.pressure.available)
|
|
82
|
+
return { pressure: "unavailable", reason: t.header.pressure.reason };
|
|
83
|
+
const spanStart = t.header.startedAt;
|
|
84
|
+
const spanEnd = t.samples.at(-1)?.t ?? spanStart;
|
|
85
|
+
const margin = th.marginSec * 1000;
|
|
86
|
+
const judged = {
|
|
87
|
+
fromMs: Math.min(w.fromMs, Math.max(w.fromMs - margin, spanStart)),
|
|
88
|
+
toMs: Math.max(w.toMs, Math.min(w.toMs + margin, spanEnd)),
|
|
89
|
+
};
|
|
90
|
+
const stats = windowStats(t.samples, judged, {
|
|
91
|
+
peakWindowSec: th.peakWindowSec,
|
|
92
|
+
ncpu: t.header.ncpu,
|
|
93
|
+
maxGapMs: maxGapMs(t),
|
|
94
|
+
});
|
|
95
|
+
if (stats.coverage < th.minCoverage) {
|
|
96
|
+
return {
|
|
97
|
+
pressure: "available",
|
|
98
|
+
contended: false,
|
|
99
|
+
reasons: [`only ${pct(stats.coverage)} of the window was sampled, too little to judge`],
|
|
100
|
+
stats,
|
|
101
|
+
};
|
|
102
|
+
}
|
|
103
|
+
const reasons = [];
|
|
104
|
+
const over = (what, v, bar) => {
|
|
105
|
+
if (v >= bar)
|
|
106
|
+
reasons.push(`${what} ${pct(v)} >= ${pct(bar)}`);
|
|
107
|
+
};
|
|
108
|
+
over("cpu some mean", stats.cpuSome.mean, th.cpuSomeMean);
|
|
109
|
+
over(`cpu some peak (${th.peakWindowSec}s)`, stats.cpuSome.peak, th.cpuSomePeak);
|
|
110
|
+
over("memory some mean", stats.memSome.mean, th.memSomeMean);
|
|
111
|
+
over(`memory full peak (${th.peakWindowSec}s)`, stats.memFull.peak, th.memFullPeak);
|
|
112
|
+
return { pressure: "available", contended: reasons.length > 0, reasons, stats };
|
|
113
|
+
}
|
|
114
|
+
// bun run turns a SIGINT/SIGKILL/SIGTERM'd child into the shell's 128+n.
|
|
115
|
+
const SIGNAL_EXITS = new Set([130, 137, 143]);
|
|
116
|
+
export function classifyRun(input) {
|
|
117
|
+
const { timeline, run, config } = input;
|
|
118
|
+
const passed = run.exitCode === 0 && run.signal === null;
|
|
119
|
+
const base = {
|
|
120
|
+
startMs: run.startMs,
|
|
121
|
+
endMs: run.endMs,
|
|
122
|
+
exitCode: run.exitCode,
|
|
123
|
+
signal: run.signal,
|
|
124
|
+
...(input.aborted ? { aborted: input.aborted } : {}),
|
|
125
|
+
};
|
|
126
|
+
const items = input.failures.map((f) => ({
|
|
127
|
+
name: f.name,
|
|
128
|
+
...(f.file ? { file: f.file } : {}),
|
|
129
|
+
fromMs: f.endMs - f.durationMs,
|
|
130
|
+
toMs: f.endMs,
|
|
131
|
+
}));
|
|
132
|
+
const killedFromOutside = !run.killedByUs && (run.signal !== null || (run.exitCode !== null && SIGNAL_EXITS.has(run.exitCode)));
|
|
133
|
+
if (killedFromOutside && !passed) {
|
|
134
|
+
items.push({
|
|
135
|
+
name: `suite killed by ${run.signal ?? `exit ${run.exitCode}`}`,
|
|
136
|
+
fromMs: Math.max(run.startMs, run.endMs - config.classify.killWindowSec * 1000),
|
|
137
|
+
toMs: run.endMs,
|
|
138
|
+
});
|
|
139
|
+
}
|
|
140
|
+
const failures = items.map((it) => {
|
|
141
|
+
const answer = contendedDuring(timeline, it, config.classify);
|
|
142
|
+
return { ...it, contended: answer.pressure === "available" && answer.contended, answer };
|
|
143
|
+
});
|
|
144
|
+
const pressure = timeline.header.pressure;
|
|
145
|
+
if (!pressure.available) {
|
|
146
|
+
const why = `pressure unavailable (${pressure.reason})`;
|
|
147
|
+
return {
|
|
148
|
+
...base,
|
|
149
|
+
status: passed ? "pass" : "fail",
|
|
150
|
+
reason: passed ? `suite passed; ${why}` : `suite failed; ${why}, so the result stands`,
|
|
151
|
+
pressure: "unavailable",
|
|
152
|
+
pressureReason: pressure.reason,
|
|
153
|
+
failures,
|
|
154
|
+
};
|
|
155
|
+
}
|
|
156
|
+
const common = { ...base, pressure: "available", failures };
|
|
157
|
+
if (input.aborted) {
|
|
158
|
+
return {
|
|
159
|
+
...common,
|
|
160
|
+
status: "uncertain",
|
|
161
|
+
reason: `stopped early: the host was contended (${input.aborted.rule}): ${input.aborted.reasons.join("; ")}`,
|
|
162
|
+
};
|
|
163
|
+
}
|
|
164
|
+
if (passed)
|
|
165
|
+
return { ...common, status: "pass", reason: "suite passed" };
|
|
166
|
+
if (failures.length === 0) {
|
|
167
|
+
return {
|
|
168
|
+
...common,
|
|
169
|
+
status: "fail",
|
|
170
|
+
reason: `the suite exited ${run.exitCode ?? run.signal} with no failing test to blame, so it is not classified`,
|
|
171
|
+
};
|
|
172
|
+
}
|
|
173
|
+
const quiet = failures.filter((f) => !f.contended);
|
|
174
|
+
if (quiet.length > 0) {
|
|
175
|
+
return {
|
|
176
|
+
...common,
|
|
177
|
+
status: "fail",
|
|
178
|
+
reason: `${quiet.length} of ${failures.length} failure(s) on a quiet host: ${quiet.map((f) => f.name).join(", ")}`,
|
|
179
|
+
};
|
|
180
|
+
}
|
|
181
|
+
return {
|
|
182
|
+
...common,
|
|
183
|
+
status: "uncertain",
|
|
184
|
+
reason: `all ${failures.length} failure(s) overlapped host contention`,
|
|
185
|
+
};
|
|
186
|
+
}
|
|
187
|
+
export function abortDecision(samples, at, cfg) {
|
|
188
|
+
if (!cfg.enabled)
|
|
189
|
+
return { abort: false };
|
|
190
|
+
const elapsed = at.nowMs - at.startMs;
|
|
191
|
+
const check = (rule, windowSec) => {
|
|
192
|
+
const stats = windowStats(samples, { fromMs: at.nowMs - windowSec * 1000, toMs: at.nowMs }, { peakWindowSec: windowSec });
|
|
193
|
+
if (stats.coverage < cfg.minCoverage)
|
|
194
|
+
return { abort: false };
|
|
195
|
+
const reasons = [];
|
|
196
|
+
const over = (what, v, bar) => {
|
|
197
|
+
if (v >= bar)
|
|
198
|
+
reasons.push(`${what} ${pct(v)} >= ${pct(bar)} over the last ${windowSec}s`);
|
|
199
|
+
};
|
|
200
|
+
over("cpu some", stats.cpuSome.mean, cfg.cpuSome);
|
|
201
|
+
over("memory some", stats.memSome.mean, cfg.memSome);
|
|
202
|
+
over("memory full", stats.memFull.mean, cfg.memFull);
|
|
203
|
+
return reasons.length ? { abort: true, rule, reasons, stats } : { abort: false };
|
|
204
|
+
};
|
|
205
|
+
if (elapsed >= cfg.earlyWindowSec * 1000 && elapsed <= cfg.earlyWithinSec * 1000) {
|
|
206
|
+
const d = check("early", cfg.earlyWindowSec);
|
|
207
|
+
if (d.abort)
|
|
208
|
+
return d;
|
|
209
|
+
}
|
|
210
|
+
if (elapsed >= cfg.sustainedWindowSec * 1000)
|
|
211
|
+
return check("sustained", cfg.sustainedWindowSec);
|
|
212
|
+
return { abort: false };
|
|
213
|
+
}
|
|
214
|
+
export function exitCodeFor(status, childCode) {
|
|
215
|
+
if (status === "pass")
|
|
216
|
+
return 0;
|
|
217
|
+
if (status === "uncertain")
|
|
218
|
+
return UNCERTAIN_EXIT;
|
|
219
|
+
return childCode !== null && childCode > 0 ? childCode : 1;
|
|
220
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@norskvideo/ctl-test-harness",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.104",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"exports": {
|
|
6
6
|
".": {
|
|
@@ -78,13 +78,18 @@
|
|
|
78
78
|
"./media-conversion": {
|
|
79
79
|
"types": "./media-conversion/index.d.ts",
|
|
80
80
|
"default": "./media-conversion/index.js"
|
|
81
|
+
},
|
|
82
|
+
"./contention": {
|
|
83
|
+
"types": "./contention/index.d.ts",
|
|
84
|
+
"default": "./contention/index.js"
|
|
81
85
|
}
|
|
82
86
|
},
|
|
83
87
|
"main": "./index.js",
|
|
84
88
|
"types": "./index.d.ts",
|
|
85
89
|
"bin": {
|
|
86
90
|
"ctl-demo": "./demo/cli.js",
|
|
87
|
-
"ctl-dev-loop": "./demo/dev-loop-cli.js"
|
|
91
|
+
"ctl-dev-loop": "./demo/dev-loop-cli.js",
|
|
92
|
+
"ctl-contention": "./contention/cli.js"
|
|
88
93
|
},
|
|
89
94
|
"dependencies": {
|
|
90
95
|
"@norskvideo/ctl-product-template-schema": "^0.1.20",
|
|
@@ -102,7 +107,7 @@
|
|
|
102
107
|
"@types/node": "^22.10.0",
|
|
103
108
|
"@norskvideo/norsk-studio": "1.30.0-2026-10-20-6fb182ff",
|
|
104
109
|
"@norskvideo/norsk-studio-built-ins": "1.30.0-2026-10-20-6fb182ff",
|
|
105
|
-
"@norskvideo/norsk-sdk": "1.3.1-2026-10-
|
|
110
|
+
"@norskvideo/norsk-sdk": "1.3.1-2026-10-22-aeac91a4"
|
|
106
111
|
},
|
|
107
112
|
"peerDependenciesMeta": {
|
|
108
113
|
"@norskvideo/norsk-studio": {
|