dungbeetle 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +105 -0
- package/NOTICE +19 -0
- package/README.md +139 -0
- package/dist/api/capture.d.ts +24 -0
- package/dist/api/capture.js +61 -0
- package/dist/baselines.d.ts +7 -0
- package/dist/baselines.js +38 -0
- package/dist/brand.d.ts +2 -0
- package/dist/brand.js +9 -0
- package/dist/capture.d.ts +15 -0
- package/dist/capture.js +7 -0
- package/dist/captures/api.d.ts +2 -0
- package/dist/captures/api.js +114 -0
- package/dist/captures/check.d.ts +2 -0
- package/dist/captures/check.js +116 -0
- package/dist/captures/desktop.d.ts +2 -0
- package/dist/captures/desktop.js +97 -0
- package/dist/captures/game.d.ts +4 -0
- package/dist/captures/game.js +266 -0
- package/dist/captures/performance.d.ts +2 -0
- package/dist/captures/performance.js +47 -0
- package/dist/captures/registry.d.ts +4 -0
- package/dist/captures/registry.js +23 -0
- package/dist/captures/terminal.d.ts +2 -0
- package/dist/captures/terminal.js +65 -0
- package/dist/captures/types.d.ts +18 -0
- package/dist/captures/types.js +1 -0
- package/dist/captures/web.d.ts +3 -0
- package/dist/captures/web.js +248 -0
- package/dist/check/capture.d.ts +15 -0
- package/dist/check/capture.js +76 -0
- package/dist/check/junit.d.ts +9 -0
- package/dist/check/junit.js +51 -0
- package/dist/check/laravel.d.ts +2 -0
- package/dist/check/laravel.js +44 -0
- package/dist/check/parsers.d.ts +12 -0
- package/dist/check/parsers.js +278 -0
- package/dist/check/schema.d.ts +2 -0
- package/dist/check/schema.js +114 -0
- package/dist/cloud.d.ts +42 -0
- package/dist/cloud.js +334 -0
- package/dist/compare/shared.d.ts +42 -0
- package/dist/compare/shared.js +115 -0
- package/dist/compare.d.ts +3 -0
- package/dist/compare.js +33 -0
- package/dist/config.d.ts +146 -0
- package/dist/config.js +382 -0
- package/dist/desktop/a11y.d.ts +18 -0
- package/dist/desktop/a11y.js +74 -0
- package/dist/desktop/capture.d.ts +13 -0
- package/dist/desktop/capture.js +80 -0
- package/dist/desktop/macos.d.ts +8 -0
- package/dist/desktop/macos.js +98 -0
- package/dist/desktop/ocr.d.ts +17 -0
- package/dist/desktop/ocr.js +99 -0
- package/dist/diff/lcs.d.ts +5 -0
- package/dist/diff/lcs.js +42 -0
- package/dist/diff/numeric.d.ts +6 -0
- package/dist/diff/numeric.js +24 -0
- package/dist/diff/pixel.d.ts +23 -0
- package/dist/diff/pixel.js +97 -0
- package/dist/diff/structural.d.ts +11 -0
- package/dist/diff/structural.js +38 -0
- package/dist/diff/text.d.ts +7 -0
- package/dist/diff/text.js +64 -0
- package/dist/diff/tree.d.ts +46 -0
- package/dist/diff/tree.js +188 -0
- package/dist/doctor.d.ts +18 -0
- package/dist/doctor.js +57 -0
- package/dist/game/capture.d.ts +24 -0
- package/dist/game/capture.js +51 -0
- package/dist/game/protocol.d.ts +30 -0
- package/dist/game/protocol.js +146 -0
- package/dist/game/walkthrough.d.ts +45 -0
- package/dist/game/walkthrough.js +85 -0
- package/dist/guards.d.ts +2 -0
- package/dist/guards.js +15 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.js +504 -0
- package/dist/json.d.ts +2 -0
- package/dist/json.js +40 -0
- package/dist/lifecycle.d.ts +14 -0
- package/dist/lifecycle.js +190 -0
- package/dist/normalization.d.ts +4 -0
- package/dist/normalization.js +27 -0
- package/dist/perf/ab.d.ts +6 -0
- package/dist/perf/ab.js +89 -0
- package/dist/perf/autocannon.d.ts +6 -0
- package/dist/perf/autocannon.js +101 -0
- package/dist/perf/capture.d.ts +7 -0
- package/dist/perf/capture.js +6 -0
- package/dist/perf/k6.d.ts +9 -0
- package/dist/perf/k6.js +44 -0
- package/dist/perf/parsers.d.ts +15 -0
- package/dist/perf/parsers.js +69 -0
- package/dist/perf/run.d.ts +8 -0
- package/dist/perf/run.js +45 -0
- package/dist/perf/toolOutput.d.ts +3 -0
- package/dist/perf/toolOutput.js +24 -0
- package/dist/reporters.d.ts +11 -0
- package/dist/reporters.js +314 -0
- package/dist/runner.d.ts +48 -0
- package/dist/runner.js +352 -0
- package/dist/snapshot.d.ts +48 -0
- package/dist/snapshot.js +37 -0
- package/dist/terminal/ansi.d.ts +21 -0
- package/dist/terminal/ansi.js +144 -0
- package/dist/terminal/capture.d.ts +30 -0
- package/dist/terminal/capture.js +91 -0
- package/dist/tty.d.ts +72 -0
- package/dist/tty.js +175 -0
- package/dist/web/domSnapshot.d.ts +27 -0
- package/dist/web/domSnapshot.js +55 -0
- package/dist/web/playwrightCapture.d.ts +16 -0
- package/dist/web/playwrightCapture.js +64 -0
- package/package.json +79 -0
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
import { spawn } from "node:child_process";
|
|
2
|
+
export async function runLifecycleCommands(lifecycle, cwd = process.cwd()) {
|
|
3
|
+
const runs = [];
|
|
4
|
+
for (const command of lifecycle.setup) {
|
|
5
|
+
runs.push(await runLifecycleCommand("setup", command, cwd));
|
|
6
|
+
}
|
|
7
|
+
for (const command of lifecycle.start) {
|
|
8
|
+
runs.push(await runLifecycleCommand("start", command, cwd));
|
|
9
|
+
}
|
|
10
|
+
if (lifecycle.wait.command) {
|
|
11
|
+
runs.push(await runLifecycleCommand("wait", lifecycle.wait.command, cwd));
|
|
12
|
+
}
|
|
13
|
+
for (const command of lifecycle.teardown) {
|
|
14
|
+
runs.push(await runLifecycleCommand("teardown", command, cwd));
|
|
15
|
+
}
|
|
16
|
+
return runs;
|
|
17
|
+
}
|
|
18
|
+
export async function startManagedLifecycle(lifecycle, cwd = process.cwd()) {
|
|
19
|
+
const runs = [];
|
|
20
|
+
const startedProcesses = [];
|
|
21
|
+
for (const command of lifecycle.setup) {
|
|
22
|
+
runs.push(await runLifecycleCommand("setup", command, cwd));
|
|
23
|
+
}
|
|
24
|
+
for (const command of lifecycle.start) {
|
|
25
|
+
const started = startLifecycleProcess(command, cwd);
|
|
26
|
+
const run = {
|
|
27
|
+
phase: "start",
|
|
28
|
+
command,
|
|
29
|
+
exitCode: null,
|
|
30
|
+
durationMs: 0,
|
|
31
|
+
pid: started.pid
|
|
32
|
+
};
|
|
33
|
+
startedProcesses.push({ process: started, run });
|
|
34
|
+
runs.push(run);
|
|
35
|
+
}
|
|
36
|
+
if (lifecycle.wait.command) {
|
|
37
|
+
runs.push(await waitForCommand(lifecycle.wait.command, cwd, lifecycle.wait.timeoutMs));
|
|
38
|
+
}
|
|
39
|
+
else if (lifecycle.wait.url) {
|
|
40
|
+
runs.push(await waitForUrl(lifecycle.wait.url, lifecycle.wait.timeoutMs));
|
|
41
|
+
}
|
|
42
|
+
return {
|
|
43
|
+
runs,
|
|
44
|
+
stop: async () => {
|
|
45
|
+
const stopRuns = [];
|
|
46
|
+
// Snapshot start-process exit state before we tear anything down. A start
|
|
47
|
+
// command that exited on its own (e.g. a server that crashed on boot) is
|
|
48
|
+
// recorded with its real exit code so the run can fail instead of being
|
|
49
|
+
// treated as a still-running background process.
|
|
50
|
+
for (const { process, run } of startedProcesses) {
|
|
51
|
+
const exit = process.exitStatus();
|
|
52
|
+
if (exit.exited) {
|
|
53
|
+
run.exitCode = exit.exitCode ?? 1;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
for (const command of lifecycle.teardown) {
|
|
57
|
+
stopRuns.push(await runLifecycleCommand("teardown", command, cwd));
|
|
58
|
+
}
|
|
59
|
+
for (const { process } of startedProcesses.reverse()) {
|
|
60
|
+
await process.stop();
|
|
61
|
+
}
|
|
62
|
+
return stopRuns;
|
|
63
|
+
}
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
function runLifecycleCommand(phase, command, cwd) {
|
|
67
|
+
return new Promise((resolve, reject) => {
|
|
68
|
+
const startedAt = performance.now();
|
|
69
|
+
const child = spawn(command, {
|
|
70
|
+
cwd,
|
|
71
|
+
shell: true,
|
|
72
|
+
stdio: "inherit"
|
|
73
|
+
});
|
|
74
|
+
child.on("error", reject);
|
|
75
|
+
child.on("close", (exitCode) => {
|
|
76
|
+
resolve({
|
|
77
|
+
phase,
|
|
78
|
+
command,
|
|
79
|
+
exitCode,
|
|
80
|
+
durationMs: Math.round(performance.now() - startedAt)
|
|
81
|
+
});
|
|
82
|
+
});
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
function startLifecycleProcess(command, cwd) {
|
|
86
|
+
const child = spawn(command, {
|
|
87
|
+
cwd,
|
|
88
|
+
detached: process.platform !== "win32",
|
|
89
|
+
shell: true,
|
|
90
|
+
stdio: "ignore"
|
|
91
|
+
});
|
|
92
|
+
child.unref();
|
|
93
|
+
let exited = false;
|
|
94
|
+
let exitCode = null;
|
|
95
|
+
child.once("exit", (code) => {
|
|
96
|
+
exited = true;
|
|
97
|
+
exitCode = code;
|
|
98
|
+
});
|
|
99
|
+
const closed = new Promise((resolve) => {
|
|
100
|
+
child.once("close", () => {
|
|
101
|
+
resolve();
|
|
102
|
+
});
|
|
103
|
+
});
|
|
104
|
+
return {
|
|
105
|
+
command,
|
|
106
|
+
pid: child.pid,
|
|
107
|
+
exitStatus: () => ({ exited, exitCode }),
|
|
108
|
+
stop: async () => {
|
|
109
|
+
if (child.killed) {
|
|
110
|
+
return;
|
|
111
|
+
}
|
|
112
|
+
if (process.platform === "win32") {
|
|
113
|
+
if (child.pid) {
|
|
114
|
+
await new Promise((resolve) => {
|
|
115
|
+
const taskkill = spawn("taskkill", ["/pid", String(child.pid), "/T", "/F"], {
|
|
116
|
+
stdio: "ignore"
|
|
117
|
+
});
|
|
118
|
+
taskkill.once("close", () => {
|
|
119
|
+
resolve();
|
|
120
|
+
});
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
else {
|
|
124
|
+
child.kill();
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
else if (child.pid) {
|
|
128
|
+
try {
|
|
129
|
+
process.kill(-child.pid, "SIGTERM");
|
|
130
|
+
}
|
|
131
|
+
catch {
|
|
132
|
+
child.kill("SIGTERM");
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
await Promise.race([closed, delay(1_000)]);
|
|
136
|
+
}
|
|
137
|
+
};
|
|
138
|
+
}
|
|
139
|
+
async function waitForCommand(command, cwd, timeoutMs) {
|
|
140
|
+
const startedAt = performance.now();
|
|
141
|
+
let lastExitCode = null;
|
|
142
|
+
while (performance.now() - startedAt <= timeoutMs) {
|
|
143
|
+
const result = await runLifecycleCommand("wait", command, cwd);
|
|
144
|
+
if (result.exitCode === 0) {
|
|
145
|
+
return {
|
|
146
|
+
...result,
|
|
147
|
+
durationMs: Math.round(performance.now() - startedAt)
|
|
148
|
+
};
|
|
149
|
+
}
|
|
150
|
+
lastExitCode = result.exitCode;
|
|
151
|
+
await delay(250);
|
|
152
|
+
}
|
|
153
|
+
return {
|
|
154
|
+
phase: "wait",
|
|
155
|
+
command,
|
|
156
|
+
exitCode: lastExitCode ?? 1,
|
|
157
|
+
durationMs: Math.round(performance.now() - startedAt)
|
|
158
|
+
};
|
|
159
|
+
}
|
|
160
|
+
async function waitForUrl(url, timeoutMs) {
|
|
161
|
+
const startedAt = performance.now();
|
|
162
|
+
while (performance.now() - startedAt <= timeoutMs) {
|
|
163
|
+
try {
|
|
164
|
+
const response = await fetch(url);
|
|
165
|
+
if (response.ok) {
|
|
166
|
+
return {
|
|
167
|
+
phase: "wait",
|
|
168
|
+
command: `GET ${url}`,
|
|
169
|
+
exitCode: 0,
|
|
170
|
+
durationMs: Math.round(performance.now() - startedAt)
|
|
171
|
+
};
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
catch {
|
|
175
|
+
// Keep polling until timeout.
|
|
176
|
+
}
|
|
177
|
+
await delay(250);
|
|
178
|
+
}
|
|
179
|
+
return {
|
|
180
|
+
phase: "wait",
|
|
181
|
+
command: `GET ${url}`,
|
|
182
|
+
exitCode: 1,
|
|
183
|
+
durationMs: Math.round(performance.now() - startedAt)
|
|
184
|
+
};
|
|
185
|
+
}
|
|
186
|
+
function delay(ms) {
|
|
187
|
+
return new Promise((resolve) => {
|
|
188
|
+
setTimeout(resolve, ms);
|
|
189
|
+
});
|
|
190
|
+
}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
// `applyMaskRules` runs once per text node, attribute, and terminal segment, so
|
|
2
|
+
// recompiling each rule's RegExp every call cost thousands of compilations per
|
|
3
|
+
// snapshot. Compile once per rules array and cache it — keyed by the array
|
|
4
|
+
// identity, which is the stable `config.normalization.masks` reference for a run.
|
|
5
|
+
// Reusing a global RegExp across `String.replace` calls is safe (replace doesn't
|
|
6
|
+
// carry `lastIndex` between calls).
|
|
7
|
+
const compiledCache = new WeakMap();
|
|
8
|
+
function compileMasks(rules) {
|
|
9
|
+
let compiled = compiledCache.get(rules);
|
|
10
|
+
if (!compiled) {
|
|
11
|
+
compiled = rules.map((rule) => ({
|
|
12
|
+
regex: new RegExp(rule.pattern, "g"),
|
|
13
|
+
replacement: rule.replacement
|
|
14
|
+
}));
|
|
15
|
+
compiledCache.set(rules, compiled);
|
|
16
|
+
}
|
|
17
|
+
return compiled;
|
|
18
|
+
}
|
|
19
|
+
export function applyMaskRules(value, rules) {
|
|
20
|
+
return compileMasks(rules).reduce((current, mask) => current.replace(mask.regex, mask.replacement), value);
|
|
21
|
+
}
|
|
22
|
+
export function normalizeLineEndings(value) {
|
|
23
|
+
return value.replace(/\r\n/g, "\n").replace(/\r/g, "\n");
|
|
24
|
+
}
|
|
25
|
+
export function collapseWhitespace(value) {
|
|
26
|
+
return value.replace(/\s+/g, " ").trim();
|
|
27
|
+
}
|
package/dist/perf/ab.js
ADDED
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
// Apache Benchmark (ab) parser for the performance kind.
|
|
2
|
+
//
|
|
3
|
+
// ab writes a plain-text summary to stdout; this normalizes it into the same
|
|
4
|
+
// `{kind: "performance", tool, metrics}` shape as k6 so comparison, tolerance,
|
|
5
|
+
// and rendering stay uniform. A target either runs `command` ("ab -n 200 -c 10
|
|
6
|
+
// <url>") or ingests a saved output via `summary`.
|
|
7
|
+
import { readToolOutput } from "./toolOutput.js";
|
|
8
|
+
function round3(value) {
|
|
9
|
+
return Math.round(value * 1000) / 1000;
|
|
10
|
+
}
|
|
11
|
+
export function normalizeAbOutput(output, options = {}) {
|
|
12
|
+
const line = (pattern) => {
|
|
13
|
+
const match = pattern.exec(output);
|
|
14
|
+
return match ? Number(match[1]) : undefined;
|
|
15
|
+
};
|
|
16
|
+
const complete = line(/^Complete requests:\s+([\d.]+)/m);
|
|
17
|
+
if (complete === undefined) {
|
|
18
|
+
throw new Error('Invalid ab output: expected a "Complete requests" line.');
|
|
19
|
+
}
|
|
20
|
+
const metrics = {
|
|
21
|
+
requests: {
|
|
22
|
+
complete,
|
|
23
|
+
failed: line(/^Failed requests:\s+([\d.]+)/m) ?? 0,
|
|
24
|
+
per_second: line(/^Requests per second:\s+([\d.]+)/m) ?? 0
|
|
25
|
+
},
|
|
26
|
+
duration_ms: {
|
|
27
|
+
mean: line(/^Time per request:\s+([\d.]+) \[ms\] \(mean\)/m) ?? 0,
|
|
28
|
+
mean_concurrent: line(/^Time per request:\s+([\d.]+) \[ms\] \(mean, across/m) ?? 0
|
|
29
|
+
}
|
|
30
|
+
};
|
|
31
|
+
const documentLength = line(/^Document Length:\s+([\d.]+)/m);
|
|
32
|
+
if (documentLength !== undefined) {
|
|
33
|
+
// Response size is signal: a changed document length means the page changed.
|
|
34
|
+
metrics.document = { length_bytes: documentLength };
|
|
35
|
+
}
|
|
36
|
+
// Connection Times table: min mean [+/-sd] median max per phase.
|
|
37
|
+
for (const row of output.matchAll(/^(Connect|Processing|Waiting|Total):\s+([\d.]+)\s+([\d.]+)\s+([\d.]+)\s+([\d.]+)\s+([\d.]+)$/gm)) {
|
|
38
|
+
metrics[`${row[1].toLowerCase()}_ms`] = {
|
|
39
|
+
min: Number(row[2]),
|
|
40
|
+
mean: Number(row[3]),
|
|
41
|
+
sd: Number(row[4]),
|
|
42
|
+
median: Number(row[5]),
|
|
43
|
+
max: Number(row[6])
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
// Percentile table: " 95% 2".
|
|
47
|
+
const percentiles = {};
|
|
48
|
+
for (const row of output.matchAll(/^\s*(\d+)%\s+(\d+)/gm)) {
|
|
49
|
+
percentiles[`p${row[1]}`] = Number(row[2]);
|
|
50
|
+
}
|
|
51
|
+
if (Object.keys(percentiles).length > 0) {
|
|
52
|
+
metrics.percentiles_ms = percentiles;
|
|
53
|
+
}
|
|
54
|
+
const allow = options.metrics ? new Set(options.metrics) : undefined;
|
|
55
|
+
const filtered = {};
|
|
56
|
+
for (const [name, stats] of Object.entries(metrics)) {
|
|
57
|
+
if (allow && !allow.has(name)) {
|
|
58
|
+
continue;
|
|
59
|
+
}
|
|
60
|
+
filtered[name] = Object.fromEntries(Object.entries(stats).map(([stat, value]) => [stat, round3(value)]));
|
|
61
|
+
}
|
|
62
|
+
return { kind: "performance", tool: "ab", metrics: filtered };
|
|
63
|
+
}
|
|
64
|
+
export const ab = {
|
|
65
|
+
tool: "ab",
|
|
66
|
+
async capture(target, options) {
|
|
67
|
+
return normalizeAbOutput(await readToolOutput(target, options), { metrics: target.metrics });
|
|
68
|
+
},
|
|
69
|
+
validate(target) {
|
|
70
|
+
return target.command || target.summary ? [] : [`must set "command" or "summary".`];
|
|
71
|
+
},
|
|
72
|
+
doctorChecks(target) {
|
|
73
|
+
return [
|
|
74
|
+
target.command || target.summary
|
|
75
|
+
? {
|
|
76
|
+
name: "performance-target",
|
|
77
|
+
severity: "pass",
|
|
78
|
+
target: target.name,
|
|
79
|
+
message: `Performance target "${target.name}" has an ${target.command ? "ab command" : "output file"}.`
|
|
80
|
+
}
|
|
81
|
+
: {
|
|
82
|
+
name: "performance-target",
|
|
83
|
+
severity: "fail",
|
|
84
|
+
target: target.name,
|
|
85
|
+
message: `Performance target "${target.name}" must set "command" or "summary".`
|
|
86
|
+
}
|
|
87
|
+
];
|
|
88
|
+
}
|
|
89
|
+
};
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import type { PerformanceSnapshot } from "./k6.js";
|
|
2
|
+
import type { PerfParser } from "./parsers.js";
|
|
3
|
+
export declare function normalizeAutocannonSummary(summary: unknown, options?: {
|
|
4
|
+
metrics?: string[];
|
|
5
|
+
}): PerformanceSnapshot;
|
|
6
|
+
export declare const autocannon: PerfParser;
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
// autocannon parser for the performance kind.
|
|
2
|
+
//
|
|
3
|
+
// `autocannon --json <url>` writes a structured summary to stdout: latency /
|
|
4
|
+
// requests / throughput stat groups (mean, stddev, min, max, percentiles) plus
|
|
5
|
+
// top-level error and status-class counters. This normalizes it into the same
|
|
6
|
+
// `{kind: "performance", tool, metrics}` shape as k6 and ab.
|
|
7
|
+
import { isRecord } from "../guards.js";
|
|
8
|
+
import { parseJsonFile } from "../json.js";
|
|
9
|
+
import { readToolOutput } from "./toolOutput.js";
|
|
10
|
+
function round3(value) {
|
|
11
|
+
return Math.round(value * 1000) / 1000;
|
|
12
|
+
}
|
|
13
|
+
export function normalizeAutocannonSummary(summary, options = {}) {
|
|
14
|
+
if (!isRecord(summary) || !isRecord(summary.latency) || !isRecord(summary.requests)) {
|
|
15
|
+
throw new Error("Invalid autocannon output: expected `latency` and `requests` objects.");
|
|
16
|
+
}
|
|
17
|
+
const metrics = {};
|
|
18
|
+
// Stat groups, verbatim (keys like p97_5 are already path-friendly).
|
|
19
|
+
for (const group of ["latency", "requests", "throughput"]) {
|
|
20
|
+
const raw = summary[group];
|
|
21
|
+
if (!isRecord(raw)) {
|
|
22
|
+
continue;
|
|
23
|
+
}
|
|
24
|
+
const stats = {};
|
|
25
|
+
for (const [key, value] of Object.entries(raw)) {
|
|
26
|
+
if (typeof value === "number" && Number.isFinite(value)) {
|
|
27
|
+
stats[key] = round3(value);
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
if (Object.keys(stats).length > 0) {
|
|
31
|
+
metrics[group] = stats;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
// Error/status counters: a regression here is signal even when timing holds.
|
|
35
|
+
const counts = {};
|
|
36
|
+
for (const key of [
|
|
37
|
+
"errors",
|
|
38
|
+
"timeouts",
|
|
39
|
+
"mismatches",
|
|
40
|
+
"non2xx",
|
|
41
|
+
"resets",
|
|
42
|
+
"1xx",
|
|
43
|
+
"2xx",
|
|
44
|
+
"3xx",
|
|
45
|
+
"4xx",
|
|
46
|
+
"5xx"
|
|
47
|
+
]) {
|
|
48
|
+
const value = summary[key];
|
|
49
|
+
if (typeof value === "number" && Number.isFinite(value)) {
|
|
50
|
+
counts[key] = value;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
if (Object.keys(counts).length > 0) {
|
|
54
|
+
metrics.counts = counts;
|
|
55
|
+
}
|
|
56
|
+
// Benchmark shape: changing duration/connections means a different test.
|
|
57
|
+
const run = {};
|
|
58
|
+
for (const key of ["duration", "connections", "pipelining"]) {
|
|
59
|
+
const value = summary[key];
|
|
60
|
+
if (typeof value === "number" && Number.isFinite(value)) {
|
|
61
|
+
run[key] = value;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
if (Object.keys(run).length > 0) {
|
|
65
|
+
metrics.run = run;
|
|
66
|
+
}
|
|
67
|
+
const allow = options.metrics ? new Set(options.metrics) : undefined;
|
|
68
|
+
const filtered = allow
|
|
69
|
+
? Object.fromEntries(Object.entries(metrics).filter(([name]) => allow.has(name)))
|
|
70
|
+
: metrics;
|
|
71
|
+
return { kind: "performance", tool: "autocannon", metrics: filtered };
|
|
72
|
+
}
|
|
73
|
+
export const autocannon = {
|
|
74
|
+
tool: "autocannon",
|
|
75
|
+
async capture(target, options) {
|
|
76
|
+
const output = await readToolOutput(target, options);
|
|
77
|
+
return normalizeAutocannonSummary(parseJsonFile(output, target.summary ?? target.command ?? "autocannon"), {
|
|
78
|
+
metrics: target.metrics
|
|
79
|
+
});
|
|
80
|
+
},
|
|
81
|
+
validate(target) {
|
|
82
|
+
return target.command || target.summary ? [] : [`must set "command" or "summary".`];
|
|
83
|
+
},
|
|
84
|
+
doctorChecks(target) {
|
|
85
|
+
return [
|
|
86
|
+
target.command || target.summary
|
|
87
|
+
? {
|
|
88
|
+
name: "performance-target",
|
|
89
|
+
severity: "pass",
|
|
90
|
+
target: target.name,
|
|
91
|
+
message: `Performance target "${target.name}" has an ${target.command ? "autocannon command" : "output file"}.`
|
|
92
|
+
}
|
|
93
|
+
: {
|
|
94
|
+
name: "performance-target",
|
|
95
|
+
severity: "fail",
|
|
96
|
+
target: target.name,
|
|
97
|
+
message: `Performance target "${target.name}" must set "command" or "summary".`
|
|
98
|
+
}
|
|
99
|
+
];
|
|
100
|
+
}
|
|
101
|
+
};
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { CaptureTarget } from "../config.js";
|
|
2
|
+
import type { PerformanceSnapshot } from "./k6.js";
|
|
3
|
+
import { type PerfCaptureOptions } from "./parsers.js";
|
|
4
|
+
export type PerformanceTarget = Extract<CaptureTarget, {
|
|
5
|
+
kind: "performance";
|
|
6
|
+
}>;
|
|
7
|
+
export declare function capturePerformance(target: PerformanceTarget, options: PerfCaptureOptions): Promise<PerformanceSnapshot>;
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import { perfParserFor } from "./parsers.js";
|
|
2
|
+
// Capture a performance snapshot via the target's tool parser (`tool`,
|
|
3
|
+
// default "k6") — see src/perf/parsers.ts for the registry.
|
|
4
|
+
export async function capturePerformance(target, options) {
|
|
5
|
+
return perfParserFor(target).capture(target, options);
|
|
6
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
export type PerformanceSnapshot = {
|
|
2
|
+
kind: "performance";
|
|
3
|
+
tool: string;
|
|
4
|
+
metrics: Record<string, Record<string, number>>;
|
|
5
|
+
};
|
|
6
|
+
export type NormalizeOptions = {
|
|
7
|
+
metrics?: string[];
|
|
8
|
+
};
|
|
9
|
+
export declare function normalizeK6Summary(summary: unknown, options?: NormalizeOptions): PerformanceSnapshot;
|
package/dist/perf/k6.js
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
// Performance snapshot model for k6.
|
|
2
|
+
//
|
|
3
|
+
// k6's `--summary-export` writes an end-of-test JSON summary whose `metrics`
|
|
4
|
+
// map holds one flat object of statistics per metric (avg, min, med, max,
|
|
5
|
+
// percentiles, count, rate, …). We normalize that into a stable, diffable
|
|
6
|
+
// shape. Because performance numbers vary run to run, comparison relies on the
|
|
7
|
+
// existing numeric-tolerance engine — perf targets should set a relative
|
|
8
|
+
// `numericTolerance` rather than expecting exact equality.
|
|
9
|
+
import { isRecord } from "../guards.js";
|
|
10
|
+
// Map k6's raw stat keys to stable, JSON-path-friendly names.
|
|
11
|
+
function normalizeStatKey(key) {
|
|
12
|
+
const percentile = /^p\((\d+(?:\.\d+)?)\)$/.exec(key);
|
|
13
|
+
return percentile ? `p${percentile[1]}` : key;
|
|
14
|
+
}
|
|
15
|
+
export function normalizeK6Summary(summary, options = {}) {
|
|
16
|
+
if (!isRecord(summary) || !isRecord(summary.metrics)) {
|
|
17
|
+
throw new Error("Invalid k6 summary: expected a `metrics` object.");
|
|
18
|
+
}
|
|
19
|
+
const allow = options.metrics ? new Set(options.metrics) : undefined;
|
|
20
|
+
const metrics = {};
|
|
21
|
+
for (const [name, raw] of Object.entries(summary.metrics)) {
|
|
22
|
+
if (allow && !allow.has(name)) {
|
|
23
|
+
continue;
|
|
24
|
+
}
|
|
25
|
+
if (!isRecord(raw)) {
|
|
26
|
+
continue;
|
|
27
|
+
}
|
|
28
|
+
const stats = {};
|
|
29
|
+
for (const [statKey, statValue] of Object.entries(raw)) {
|
|
30
|
+
if (typeof statValue === "number" && Number.isFinite(statValue)) {
|
|
31
|
+
// Round to 3 decimals to keep baselines readable; the tolerance engine
|
|
32
|
+
// absorbs the meaningful run-to-run variance.
|
|
33
|
+
stats[normalizeStatKey(statKey)] = round3(statValue);
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
if (Object.keys(stats).length > 0) {
|
|
37
|
+
metrics[name] = stats;
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
return { kind: "performance", tool: "k6", metrics };
|
|
41
|
+
}
|
|
42
|
+
function round3(value) {
|
|
43
|
+
return Math.round(value * 1000) / 1000;
|
|
44
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { DoctorCheck } from "../doctor.js";
|
|
2
|
+
import { type PerformanceSnapshot } from "./k6.js";
|
|
3
|
+
import type { PerformanceTarget } from "./capture.js";
|
|
4
|
+
export type PerfCaptureOptions = {
|
|
5
|
+
cwd: string;
|
|
6
|
+
timeoutMs: number;
|
|
7
|
+
};
|
|
8
|
+
export interface PerfParser {
|
|
9
|
+
tool: string;
|
|
10
|
+
capture(target: PerformanceTarget, options: PerfCaptureOptions): Promise<PerformanceSnapshot>;
|
|
11
|
+
validate(target: PerformanceTarget): string[];
|
|
12
|
+
doctorChecks(target: PerformanceTarget): DoctorCheck[];
|
|
13
|
+
}
|
|
14
|
+
export declare const perfParsers: Record<string, PerfParser>;
|
|
15
|
+
export declare function perfParserFor(target: PerformanceTarget): PerfParser;
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
// Tool-parser registry for the performance kind.
|
|
2
|
+
//
|
|
3
|
+
// One capture kind can host several tool parsers: each parser owns how to
|
|
4
|
+
// obtain one tool's raw output and normalize it into the shared
|
|
5
|
+
// `{kind: "performance", tool, metrics}` shape, so comparison, tolerance, and
|
|
6
|
+
// rendering stay uniform across tools. A target selects its parser via
|
|
7
|
+
// `tool` (default "k6"). Adding a tool is one parser object plus one registry
|
|
8
|
+
// line here — the capture/validate/doctor dispatchers in
|
|
9
|
+
// `src/captures/performance.ts` route through this registry.
|
|
10
|
+
import { readFile } from "node:fs/promises";
|
|
11
|
+
import path from "node:path";
|
|
12
|
+
import { parseJsonFile } from "../json.js";
|
|
13
|
+
import { ab } from "./ab.js";
|
|
14
|
+
import { autocannon } from "./autocannon.js";
|
|
15
|
+
import { normalizeK6Summary } from "./k6.js";
|
|
16
|
+
import { runK6 } from "./run.js";
|
|
17
|
+
const k6 = {
|
|
18
|
+
tool: "k6",
|
|
19
|
+
async capture(target, options) {
|
|
20
|
+
let summary;
|
|
21
|
+
if (target.summary) {
|
|
22
|
+
const summaryPath = path.resolve(options.cwd, target.summary);
|
|
23
|
+
summary = parseJsonFile(await readFile(summaryPath, "utf8"), summaryPath);
|
|
24
|
+
}
|
|
25
|
+
else if (target.script) {
|
|
26
|
+
summary = await runK6({
|
|
27
|
+
script: path.resolve(options.cwd, target.script),
|
|
28
|
+
k6Path: target.k6Path,
|
|
29
|
+
cwd: options.cwd,
|
|
30
|
+
env: target.env,
|
|
31
|
+
timeoutMs: target.timeoutMs ?? options.timeoutMs
|
|
32
|
+
});
|
|
33
|
+
}
|
|
34
|
+
else {
|
|
35
|
+
throw new Error(`Performance target "${target.name}" must provide a "script" or "summary".`);
|
|
36
|
+
}
|
|
37
|
+
return normalizeK6Summary(summary, { metrics: target.metrics });
|
|
38
|
+
},
|
|
39
|
+
validate(target) {
|
|
40
|
+
return target.script || target.summary ? [] : [`must set "script" or "summary".`];
|
|
41
|
+
},
|
|
42
|
+
doctorChecks(target) {
|
|
43
|
+
return [
|
|
44
|
+
target.script || target.summary
|
|
45
|
+
? {
|
|
46
|
+
name: "performance-target",
|
|
47
|
+
severity: "pass",
|
|
48
|
+
target: target.name,
|
|
49
|
+
message: `Performance target "${target.name}" has a ${target.script ? "k6 script" : "summary file"}.`
|
|
50
|
+
}
|
|
51
|
+
: {
|
|
52
|
+
name: "performance-target",
|
|
53
|
+
severity: "fail",
|
|
54
|
+
target: target.name,
|
|
55
|
+
message: `Performance target "${target.name}" must set "script" or "summary".`
|
|
56
|
+
}
|
|
57
|
+
];
|
|
58
|
+
}
|
|
59
|
+
};
|
|
60
|
+
export const perfParsers = { k6, ab, autocannon };
|
|
61
|
+
export function perfParserFor(target) {
|
|
62
|
+
const tool = target.tool ?? "k6";
|
|
63
|
+
const parser = perfParsers[tool];
|
|
64
|
+
if (!parser) {
|
|
65
|
+
throw new Error(`Performance target "${target.name}" uses unknown tool "${tool}". ` +
|
|
66
|
+
`Available: ${Object.keys(perfParsers).join(", ")}.`);
|
|
67
|
+
}
|
|
68
|
+
return parser;
|
|
69
|
+
}
|
package/dist/perf/run.js
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { execFile } from "node:child_process";
|
|
2
|
+
import { mkdtemp, readFile, rm } from "node:fs/promises";
|
|
3
|
+
import { tmpdir } from "node:os";
|
|
4
|
+
import path from "node:path";
|
|
5
|
+
import { promisify } from "node:util";
|
|
6
|
+
import { isErrno } from "../guards.js";
|
|
7
|
+
import { parseJsonFile } from "../json.js";
|
|
8
|
+
const execFileAsync = promisify(execFile);
|
|
9
|
+
// Run a k6 script and return its parsed summary-export JSON. k6 must be
|
|
10
|
+
// installed and on PATH (like the optional Playwright browser); it is not a
|
|
11
|
+
// bundled dependency.
|
|
12
|
+
export async function runK6(options) {
|
|
13
|
+
const bin = options.k6Path ?? "k6";
|
|
14
|
+
const dir = await mkdtemp(path.join(tmpdir(), "dungbeetle-k6-"));
|
|
15
|
+
const summaryPath = path.join(dir, "summary.json");
|
|
16
|
+
try {
|
|
17
|
+
try {
|
|
18
|
+
await execFileAsync(bin, ["run", "--summary-export", summaryPath, options.script], {
|
|
19
|
+
cwd: options.cwd,
|
|
20
|
+
env: { ...process.env, ...options.env },
|
|
21
|
+
timeout: options.timeoutMs ?? 0,
|
|
22
|
+
maxBuffer: 16 * 1024 * 1024
|
|
23
|
+
});
|
|
24
|
+
}
|
|
25
|
+
catch (error) {
|
|
26
|
+
if (isErrno(error, "ENOENT")) {
|
|
27
|
+
throw new Error(`Could not run k6 ("${bin}"); is it installed and on PATH?`);
|
|
28
|
+
}
|
|
29
|
+
// k6 exits non-zero when a threshold is breached but still writes the
|
|
30
|
+
// summary. We snapshot metrics rather than enforce k6 thresholds, so only
|
|
31
|
+
// a missing summary (handled below) is fatal.
|
|
32
|
+
}
|
|
33
|
+
let raw;
|
|
34
|
+
try {
|
|
35
|
+
raw = await readFile(summaryPath, "utf8");
|
|
36
|
+
}
|
|
37
|
+
catch {
|
|
38
|
+
throw new Error(`k6 did not produce a summary for ${options.script}.`);
|
|
39
|
+
}
|
|
40
|
+
return parseJsonFile(raw, summaryPath);
|
|
41
|
+
}
|
|
42
|
+
finally {
|
|
43
|
+
await rm(dir, { recursive: true, force: true });
|
|
44
|
+
}
|
|
45
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
// Shared run-or-ingest for command-driven performance tools (ab, autocannon):
|
|
2
|
+
// read a saved output file via `summary`, or run `command` and return stdout.
|
|
3
|
+
import { readFile } from "node:fs/promises";
|
|
4
|
+
import path from "node:path";
|
|
5
|
+
import { runShellCommand } from "../terminal/capture.js";
|
|
6
|
+
export async function readToolOutput(target, options) {
|
|
7
|
+
if (target.summary) {
|
|
8
|
+
return readFile(path.resolve(options.cwd, target.summary), "utf8");
|
|
9
|
+
}
|
|
10
|
+
if (!target.command) {
|
|
11
|
+
throw new Error(`Performance target "${target.name}" must provide a "command" or "summary".`);
|
|
12
|
+
}
|
|
13
|
+
const result = await runShellCommand({
|
|
14
|
+
command: target.command,
|
|
15
|
+
cwd: options.cwd,
|
|
16
|
+
timeoutMs: target.timeoutMs ?? options.timeoutMs
|
|
17
|
+
});
|
|
18
|
+
if (result.exitCode !== 0) {
|
|
19
|
+
const detail = result.stderr.trim();
|
|
20
|
+
throw new Error(`Performance target "${target.name}": \`${target.command}\` exited with ` +
|
|
21
|
+
`${result.exitCode ?? `signal ${String(result.signal)}`}${detail ? `:\n${detail}` : "."}`);
|
|
22
|
+
}
|
|
23
|
+
return result.stdout;
|
|
24
|
+
}
|