@elench/testkit 0.1.53 → 0.1.54
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/cli/presentation/code-frames.mjs +57 -0
- package/lib/cli/presentation/code-frames.test.mjs +71 -0
- package/lib/cli/presentation/colors.mjs +29 -0
- package/lib/cli/presentation/run-reporter.mjs +16 -7
- package/lib/cli/viewer.mjs +109 -4
- package/lib/known-failures/index.mjs +1 -1
- package/lib/known-failures/index.test.mjs +46 -0
- package/lib/runner/default-runtime-errors.mjs +66 -0
- package/lib/runner/default-runtime-runner.mjs +8 -1
- package/lib/runner/failure-details.mjs +31 -0
- package/lib/runner/failure-details.test.mjs +51 -0
- package/lib/runner/formatting.mjs +114 -4
- package/lib/runner/formatting.test.mjs +77 -0
- package/lib/runner/logs.mjs +17 -0
- package/lib/runner/orchestrator.mjs +10 -1
- package/lib/runner/triage.mjs +67 -0
- package/lib/runtime/index.d.ts +60 -0
- package/lib/runtime/index.mjs +12 -0
- package/lib/runtime-src/k6/checks.js +45 -12
- package/lib/runtime-src/k6/http-assertions.js +214 -0
- package/lib/runtime-src/k6/http.js +261 -13
- package/lib/runtime-src/k6/suite.js +46 -1
- package/package.json +3 -1
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import fs from "fs";
|
|
2
|
+
import path from "path";
|
|
3
|
+
import { codeFrameColumns } from "@babel/code-frame";
|
|
4
|
+
import { createColors } from "picocolors";
|
|
5
|
+
|
|
6
|
+
const pc = createColors(Boolean(process.stdout?.isTTY || process.env.FORCE_COLOR));
|
|
7
|
+
|
|
8
|
+
export function findFailureLocation(detail = {}, fallbackError = "") {
|
|
9
|
+
if (detail?.location?.path && Number.isFinite(detail?.location?.line)) {
|
|
10
|
+
return detail.location;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
const stack = detail?.stack || fallbackError;
|
|
14
|
+
if (!stack) return null;
|
|
15
|
+
const match = String(stack).match(/(\/[^\s:()]+):(\d+):(\d+)/);
|
|
16
|
+
if (!match) return null;
|
|
17
|
+
return {
|
|
18
|
+
path: match[1],
|
|
19
|
+
line: Number(match[2]),
|
|
20
|
+
column: Number(match[3]),
|
|
21
|
+
};
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export function renderCodeFrame(location, options = {}) {
|
|
25
|
+
if (!location?.path || !Number.isFinite(location.line)) return [];
|
|
26
|
+
if (!fs.existsSync(location.path)) return [];
|
|
27
|
+
|
|
28
|
+
try {
|
|
29
|
+
const source = fs.readFileSync(location.path, "utf8");
|
|
30
|
+
const frame = codeFrameColumns(
|
|
31
|
+
source,
|
|
32
|
+
{
|
|
33
|
+
start: {
|
|
34
|
+
line: location.line,
|
|
35
|
+
column: Math.max(1, Number(location.column) || 1),
|
|
36
|
+
},
|
|
37
|
+
},
|
|
38
|
+
{
|
|
39
|
+
highlightCode: Boolean(process.stdout?.isTTY || process.env.FORCE_COLOR),
|
|
40
|
+
linesAbove: options.linesAbove ?? 2,
|
|
41
|
+
linesBelow: options.linesBelow ?? 2,
|
|
42
|
+
}
|
|
43
|
+
);
|
|
44
|
+
const label = `${options.label || "code"}: ${formatLocation(location, options.cwd)}`;
|
|
45
|
+
return [pc.bold(label), ...frame.split(/\r?\n/)];
|
|
46
|
+
} catch {
|
|
47
|
+
return [];
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export function formatLocation(location, cwd = process.cwd()) {
|
|
52
|
+
if (!location?.path || !Number.isFinite(location.line)) return null;
|
|
53
|
+
const relativePath = path.isAbsolute(location.path)
|
|
54
|
+
? path.relative(cwd, location.path) || path.basename(location.path)
|
|
55
|
+
: location.path;
|
|
56
|
+
return `${relativePath}:${location.line}:${location.column || 1}`;
|
|
57
|
+
}
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import fs from "fs";
|
|
2
|
+
import os from "os";
|
|
3
|
+
import path from "path";
|
|
4
|
+
import { afterEach, describe, expect, it } from "vitest";
|
|
5
|
+
import { findFailureLocation, formatLocation, renderCodeFrame } from "./code-frames.mjs";
|
|
6
|
+
|
|
7
|
+
const cleanups = [];
|
|
8
|
+
|
|
9
|
+
afterEach(() => {
|
|
10
|
+
while (cleanups.length > 0) {
|
|
11
|
+
cleanups.pop()();
|
|
12
|
+
}
|
|
13
|
+
});
|
|
14
|
+
|
|
15
|
+
describe("code frame presentation", () => {
|
|
16
|
+
it("finds a location from structured failure metadata", () => {
|
|
17
|
+
expect(
|
|
18
|
+
findFailureLocation({
|
|
19
|
+
location: {
|
|
20
|
+
path: "/tmp/example.ts",
|
|
21
|
+
line: 10,
|
|
22
|
+
column: 2,
|
|
23
|
+
},
|
|
24
|
+
})
|
|
25
|
+
).toEqual({
|
|
26
|
+
path: "/tmp/example.ts",
|
|
27
|
+
line: 10,
|
|
28
|
+
column: 2,
|
|
29
|
+
});
|
|
30
|
+
});
|
|
31
|
+
|
|
32
|
+
it("renders a code frame for a local file", () => {
|
|
33
|
+
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "testkit-code-frame-"));
|
|
34
|
+
cleanups.push(() => fs.rmSync(tempDir, { recursive: true, force: true }));
|
|
35
|
+
const filePath = path.join(tempDir, "example.ts");
|
|
36
|
+
fs.writeFileSync(
|
|
37
|
+
filePath,
|
|
38
|
+
[
|
|
39
|
+
"export function run() {",
|
|
40
|
+
" const value = 1;",
|
|
41
|
+
" return value + missing;",
|
|
42
|
+
"}",
|
|
43
|
+
].join("\n")
|
|
44
|
+
);
|
|
45
|
+
|
|
46
|
+
const lines = renderCodeFrame(
|
|
47
|
+
{
|
|
48
|
+
path: filePath,
|
|
49
|
+
line: 3,
|
|
50
|
+
column: 18,
|
|
51
|
+
},
|
|
52
|
+
{ cwd: tempDir }
|
|
53
|
+
);
|
|
54
|
+
|
|
55
|
+
expect(lines.join("\n")).toContain("example.ts:3:18");
|
|
56
|
+
expect(lines.join("\n")).toContain("return value + missing;");
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
it("formats locations relative to the cwd", () => {
|
|
60
|
+
expect(
|
|
61
|
+
formatLocation(
|
|
62
|
+
{
|
|
63
|
+
path: "/tmp/example.ts",
|
|
64
|
+
line: 7,
|
|
65
|
+
column: 3,
|
|
66
|
+
},
|
|
67
|
+
"/tmp"
|
|
68
|
+
)
|
|
69
|
+
).toBe("example.ts:7:3");
|
|
70
|
+
});
|
|
71
|
+
});
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { createColors } from "picocolors";
|
|
2
|
+
|
|
3
|
+
const pc = createColors(Boolean(process.stdout?.isTTY || process.env.FORCE_COLOR));
|
|
4
|
+
|
|
5
|
+
export function colorStatus(status) {
|
|
6
|
+
if (status === "PASS") return pc.green(status);
|
|
7
|
+
if (status === "FAIL") return pc.red(status);
|
|
8
|
+
if (status === "SKIP") return pc.yellow(status);
|
|
9
|
+
if (status === "RUN") return pc.cyan(status);
|
|
10
|
+
return status;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
export function dim(text) {
|
|
14
|
+
return pc.dim(text);
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export function colorResultLine(line) {
|
|
18
|
+
if (/^Result: PASSED\b/.test(line)) return line.replace("PASSED", pc.green("PASSED"));
|
|
19
|
+
if (/^Result: FAILED\b/.test(line)) return line.replace("FAILED", pc.red("FAILED"));
|
|
20
|
+
return line;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export function colorSectionLine(line) {
|
|
24
|
+
if (line === "Failures:" || line === "Runtime Errors:" || line === "Known-failure issues:" || line === "Triage:") {
|
|
25
|
+
return pc.bold(line);
|
|
26
|
+
}
|
|
27
|
+
if (/^Summary: /.test(line)) return line.replace("Summary:", `${pc.bold("Summary:")}`);
|
|
28
|
+
return colorResultLine(line);
|
|
29
|
+
}
|
|
@@ -4,6 +4,7 @@ import {
|
|
|
4
4
|
buildDebugRunSummaryLines,
|
|
5
5
|
formatDuration,
|
|
6
6
|
} from "../../runner/formatting.mjs";
|
|
7
|
+
import { colorSectionLine, colorStatus, dim } from "./colors.mjs";
|
|
7
8
|
|
|
8
9
|
export function createRunReporter({ outputMode = "compact", stdout = process.stdout, stderr = process.stderr } = {}) {
|
|
9
10
|
const mode = outputMode || "compact";
|
|
@@ -30,29 +31,32 @@ export function createRunReporter({ outputMode = "compact", stdout = process.std
|
|
|
30
31
|
stdout.write(`Starting ${config.runtimeLabel}:${config.name}: ${command}\n`);
|
|
31
32
|
},
|
|
32
33
|
serviceSkipped(config, reason) {
|
|
33
|
-
stdout.write(
|
|
34
|
+
stdout.write(`${colorStatus("SKIP")} ${config.name} ${reason}\n`);
|
|
34
35
|
},
|
|
35
36
|
plannedSkip(entry) {
|
|
36
37
|
stdout.write(
|
|
37
|
-
|
|
38
|
+
`${colorStatus("SKIP")} ${entry.serviceName} ${entry.type} ${normalizePath(entry.file)} ${dim("0s")} ${shortenMessage(entry.reason || "skipped")}\n`
|
|
38
39
|
);
|
|
39
40
|
},
|
|
40
41
|
taskStarted(task, targetConfig) {
|
|
41
42
|
if (mode !== "debug") return;
|
|
42
|
-
stdout.write(
|
|
43
|
+
stdout.write(`${colorStatus("RUN").padEnd(12)} ${targetConfig.name} ${task.type} ${task.file}\n`);
|
|
43
44
|
},
|
|
44
45
|
taskFinished(task, outcome) {
|
|
45
46
|
if (mode === "json") return;
|
|
46
47
|
const status = outcome.status === "skipped" ? "SKIP" : outcome.failed ? "FAIL" : "PASS";
|
|
47
48
|
const duration = formatDuration(outcome.durationMs || 0);
|
|
49
|
+
const primaryFailure = firstFailureDetail(outcome);
|
|
50
|
+
const preferredFailure =
|
|
51
|
+
primaryFailure && isThresholdWrapperMessage(outcome.error) ? primaryFailure : outcome.error || primaryFailure;
|
|
48
52
|
const detail =
|
|
49
53
|
status === "FAIL"
|
|
50
|
-
? ` ${shortenMessage(
|
|
54
|
+
? ` ${shortenMessage(preferredFailure || "failed")}`
|
|
51
55
|
: outcome.status === "not_run"
|
|
52
56
|
? ` ${shortenMessage(outcome.reason || "not run")}`
|
|
53
57
|
: "";
|
|
54
58
|
stdout.write(
|
|
55
|
-
`${status} ${task.serviceName} ${displayTaskType(task)} ${normalizePath(task.file)} ${duration}${detail}\n`
|
|
59
|
+
`${colorStatus(status)} ${task.serviceName} ${displayTaskType(task)} ${normalizePath(task.file)} ${dim(duration)}${detail}\n`
|
|
56
60
|
);
|
|
57
61
|
},
|
|
58
62
|
telemetry(message) {
|
|
@@ -64,7 +68,7 @@ export function createRunReporter({ outputMode = "compact", stdout = process.std
|
|
|
64
68
|
mode === "debug"
|
|
65
69
|
? buildDebugRunSummaryLines(results, durationMs, knownFailureIssueValidation)
|
|
66
70
|
: buildCompactRunSummaryLines(results, durationMs, knownFailureIssueValidation);
|
|
67
|
-
for (const line of lines) stdout.write(`${line}\n`);
|
|
71
|
+
for (const line of lines) stdout.write(`${colorSectionLine(line)}\n`);
|
|
68
72
|
},
|
|
69
73
|
error(message) {
|
|
70
74
|
stderr.write(`${message}\n`);
|
|
@@ -83,7 +87,12 @@ function shortenMessage(message) {
|
|
|
83
87
|
}
|
|
84
88
|
|
|
85
89
|
function firstFailureDetail(outcome) {
|
|
86
|
-
|
|
90
|
+
const detail = outcome.failureDetails?.[0];
|
|
91
|
+
return detail?.message || detail?.title || null;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
function isThresholdWrapperMessage(message) {
|
|
95
|
+
return /Default runtime thresholds failed:/.test(String(message || ""));
|
|
87
96
|
}
|
|
88
97
|
|
|
89
98
|
function normalizePath(filePath) {
|
package/lib/cli/viewer.mjs
CHANGED
|
@@ -1,7 +1,12 @@
|
|
|
1
1
|
import fs from "fs";
|
|
2
2
|
import path from "path";
|
|
3
3
|
import { formatDuration } from "../runner/formatting.mjs";
|
|
4
|
-
import { readLogTail } from "../runner/logs.mjs";
|
|
4
|
+
import { findLogSliceByRequestId, readLogTail } from "../runner/logs.mjs";
|
|
5
|
+
import {
|
|
6
|
+
findFailureLocation,
|
|
7
|
+
formatLocation,
|
|
8
|
+
renderCodeFrame,
|
|
9
|
+
} from "./presentation/code-frames.mjs";
|
|
5
10
|
|
|
6
11
|
export function loadLatestRunArtifact(productDir) {
|
|
7
12
|
const artifactPath = path.join(productDir, ".testkit", "results", "latest.json");
|
|
@@ -63,6 +68,8 @@ export function collectArtifactEntries(productDir, runArtifact, selector = null,
|
|
|
63
68
|
|
|
64
69
|
export function formatFileDetail(productDir, runArtifact, subject, options = {}) {
|
|
65
70
|
const lines = [];
|
|
71
|
+
const failureDetails = subject.file.failureDetails || [];
|
|
72
|
+
const primaryDetail = rankFailureDetails(failureDetails)[0] || null;
|
|
66
73
|
lines.push(`File: ${subject.file.path}`);
|
|
67
74
|
lines.push(`Service: ${subject.service.name}`);
|
|
68
75
|
lines.push(`Suite: ${subject.suite.type}:${subject.suite.name}`);
|
|
@@ -70,15 +77,37 @@ export function formatFileDetail(productDir, runArtifact, subject, options = {})
|
|
|
70
77
|
lines.push(`Duration: ${formatDuration(subject.file.durationMs || 0)}`);
|
|
71
78
|
if (subject.file.error) lines.push(`Error: ${subject.file.error}`);
|
|
72
79
|
|
|
73
|
-
if (
|
|
80
|
+
if (failureDetails.length > 0) {
|
|
74
81
|
lines.push("");
|
|
75
82
|
lines.push("Failure Details:");
|
|
76
|
-
for (const detail of
|
|
83
|
+
for (const detail of rankFailureDetails(failureDetails).slice(0, options.failureLimit || 5)) {
|
|
77
84
|
lines.push(` ${detail.title}`);
|
|
78
85
|
if (detail.message) lines.push(` ${detail.message}`);
|
|
86
|
+
const requestLine = formatRequestLine(detail);
|
|
87
|
+
if (requestLine) lines.push(` ${requestLine}`);
|
|
88
|
+
const responseLine = formatResponseLine(detail);
|
|
89
|
+
if (responseLine) lines.push(` ${responseLine}`);
|
|
90
|
+
const location = findFailureLocation(detail, subject.file.error || "");
|
|
91
|
+
if (location) lines.push(` at ${formatLocation(location, productDir)}`);
|
|
79
92
|
}
|
|
80
93
|
}
|
|
81
94
|
|
|
95
|
+
const codeFrame = renderCodeFrame(findFailureLocation(primaryDetail, subject.file.error || ""), {
|
|
96
|
+
cwd: productDir,
|
|
97
|
+
});
|
|
98
|
+
if (codeFrame.length > 0) {
|
|
99
|
+
lines.push("");
|
|
100
|
+
lines.push("Code Frame:");
|
|
101
|
+
for (const line of codeFrame) lines.push(` ${line}`);
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
if (subject.file.triage) {
|
|
105
|
+
lines.push("");
|
|
106
|
+
lines.push("Triage:");
|
|
107
|
+
const triageLines = formatTriage(subject.file.triage);
|
|
108
|
+
for (const line of triageLines) lines.push(` ${line}`);
|
|
109
|
+
}
|
|
110
|
+
|
|
82
111
|
const artifacts = collectArtifactEntries(productDir, runArtifact, subject.file.path, subject.service.name);
|
|
83
112
|
if (artifacts.length > 0) {
|
|
84
113
|
lines.push("");
|
|
@@ -100,7 +129,12 @@ export function formatFileDetail(productDir, runArtifact, subject, options = {})
|
|
|
100
129
|
for (const logRef of logRefs) {
|
|
101
130
|
lines.push(` ${logRef.runtimeLabel}`);
|
|
102
131
|
lines.push(` ${logRef.path}`);
|
|
103
|
-
const
|
|
132
|
+
const logPath = path.join(productDir, logRef.path);
|
|
133
|
+
const requestId = primaryDetail?.request?.requestId || null;
|
|
134
|
+
const tail =
|
|
135
|
+
requestId && findLogSliceByRequestId(logPath, requestId, 2).length > 0
|
|
136
|
+
? findLogSliceByRequestId(logPath, requestId, 2)
|
|
137
|
+
: readLogTail(logPath, options.logTail || 12);
|
|
104
138
|
for (const line of tail.slice(-Math.max(0, options.logTail || 12))) {
|
|
105
139
|
lines.push(` ${line}`);
|
|
106
140
|
}
|
|
@@ -119,6 +153,9 @@ export function formatArtifactPreview(payload, maxLines = 6) {
|
|
|
119
153
|
if (payload.kind === "agentic-query") {
|
|
120
154
|
return formatAgenticArtifact(payload, maxLines);
|
|
121
155
|
}
|
|
156
|
+
if (payload.kind === "testkit.http-traces") {
|
|
157
|
+
return formatHttpTraceArtifact(payload, maxLines);
|
|
158
|
+
}
|
|
122
159
|
if (payload.contentType === "text/plain" && typeof payload.data?.text === "string") {
|
|
123
160
|
return payload.data.text
|
|
124
161
|
.split(/\r?\n/)
|
|
@@ -145,6 +182,74 @@ function formatAgenticArtifact(payload, maxLines) {
|
|
|
145
182
|
return lines.slice(0, maxLines);
|
|
146
183
|
}
|
|
147
184
|
|
|
185
|
+
function formatHttpTraceArtifact(payload, maxLines) {
|
|
186
|
+
const traces = Array.isArray(payload.data?.traces) ? payload.data.traces : [];
|
|
187
|
+
const lines = [];
|
|
188
|
+
for (const trace of traces.slice(0, maxLines)) {
|
|
189
|
+
lines.push(
|
|
190
|
+
`${trace.method} ${trace.path} -> ${trace.response?.status ?? "?"}${trace.requestId ? ` [${trace.requestId}]` : ""}`
|
|
191
|
+
);
|
|
192
|
+
}
|
|
193
|
+
return lines;
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
function rankFailureDetails(details) {
|
|
197
|
+
return [...(Array.isArray(details) ? details : [])].sort((left, right) => {
|
|
198
|
+
return failureDetailRank(left) - failureDetailRank(right) || String(left?.key || "").localeCompare(String(right?.key || ""));
|
|
199
|
+
});
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
function failureDetailRank(detail) {
|
|
203
|
+
if (detail?.kind === "http-assertion") return 1;
|
|
204
|
+
if (detail?.request && detail?.response) return 2;
|
|
205
|
+
if (detail?.location || detail?.stack) return 3;
|
|
206
|
+
if (detail?.message) return 4;
|
|
207
|
+
return 5;
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
function formatRequestLine(detail) {
|
|
211
|
+
const method = detail?.request?.method;
|
|
212
|
+
const path = detail?.request?.path;
|
|
213
|
+
if (!method || !path) return null;
|
|
214
|
+
const requestId = detail?.request?.requestId;
|
|
215
|
+
return requestId ? `request: ${method} ${path} [${requestId}]` : `request: ${method} ${path}`;
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
function formatResponseLine(detail) {
|
|
219
|
+
if (!detail?.response) return null;
|
|
220
|
+
const parts = [`response: ${detail.response.status ?? "?"}`];
|
|
221
|
+
if (detail.response.contentType) parts.push(detail.response.contentType);
|
|
222
|
+
if (detail.response.bodyPreview) parts.push(detail.response.bodyPreview);
|
|
223
|
+
return parts.join(" ");
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
function formatTriage(triage) {
|
|
227
|
+
const lines = [`status: ${triage.status}`];
|
|
228
|
+
if (triage.classifications?.length) {
|
|
229
|
+
lines.push(`classification: ${triage.classifications.join(", ")}`);
|
|
230
|
+
}
|
|
231
|
+
if (triage.availability?.mode) {
|
|
232
|
+
lines.push(
|
|
233
|
+
`validation: ${triage.availability.mode}${triage.availability.reason ? ` (${triage.availability.reason})` : ""}`
|
|
234
|
+
);
|
|
235
|
+
}
|
|
236
|
+
for (const entry of triage.entries || []) {
|
|
237
|
+
lines.push(`issue: ${entry.issue.repo}#${entry.issue.number}`);
|
|
238
|
+
if (entry.issue.url) lines.push(`url: ${entry.issue.url}`);
|
|
239
|
+
if (entry.github?.state) {
|
|
240
|
+
lines.push(`github state: ${entry.github.state}${entry.github.cached ? " (cache)" : ""}`);
|
|
241
|
+
}
|
|
242
|
+
if (entry.validationStatus) lines.push(`validation status: ${entry.validationStatus}`);
|
|
243
|
+
if (entry.findings?.length) {
|
|
244
|
+
for (const finding of entry.findings.slice(0, 3)) {
|
|
245
|
+
lines.push(`finding: ${finding.message}`);
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
break;
|
|
249
|
+
}
|
|
250
|
+
return lines;
|
|
251
|
+
}
|
|
252
|
+
|
|
148
253
|
function collectFiles(runArtifact, serviceFilter = null) {
|
|
149
254
|
const files = [];
|
|
150
255
|
for (const service of runArtifact.services || []) {
|
|
@@ -194,7 +194,7 @@ export function matchesKnownFailureMatch(match, fileSummary) {
|
|
|
194
194
|
if (match.path !== fileSummary.path) return false;
|
|
195
195
|
if (match.failureKey) {
|
|
196
196
|
const failureKeys = Array.isArray(fileSummary.failureDetails)
|
|
197
|
-
? fileSummary.failureDetails.
|
|
197
|
+
? fileSummary.failureDetails.flatMap((detail) => [detail?.key, detail?.title].filter(Boolean))
|
|
198
198
|
: [];
|
|
199
199
|
if (!failureKeys.includes(match.failureKey)) return false;
|
|
200
200
|
}
|
|
@@ -59,6 +59,52 @@ describe("known failures core", () => {
|
|
|
59
59
|
expect(matches[0].id).toBe("bad-message");
|
|
60
60
|
});
|
|
61
61
|
|
|
62
|
+
it("matches failureKey against a detail title when the key becomes richer", () => {
|
|
63
|
+
const document = normalizeKnownFailuresDocument({
|
|
64
|
+
schemaVersion: 1,
|
|
65
|
+
entries: [
|
|
66
|
+
{
|
|
67
|
+
id: "missing-route",
|
|
68
|
+
title: "Missing route bug",
|
|
69
|
+
classification: "product_bug",
|
|
70
|
+
state: "open",
|
|
71
|
+
issue: {
|
|
72
|
+
repo: "acme/repo",
|
|
73
|
+
number: 13,
|
|
74
|
+
url: "https://github.com/acme/repo/issues/13",
|
|
75
|
+
},
|
|
76
|
+
description: "Wrong status code",
|
|
77
|
+
whyFailing: "The route returns 404",
|
|
78
|
+
lastReviewedAt: "2026-04-28",
|
|
79
|
+
matches: [
|
|
80
|
+
{
|
|
81
|
+
service: "api",
|
|
82
|
+
type: "int",
|
|
83
|
+
path: "__testkit__/health/http-failure.int.testkit.ts",
|
|
84
|
+
failureKey: "GET /missing returns 200",
|
|
85
|
+
},
|
|
86
|
+
],
|
|
87
|
+
},
|
|
88
|
+
],
|
|
89
|
+
});
|
|
90
|
+
|
|
91
|
+
const matches = findMatchingKnownFailureEntries(document, {
|
|
92
|
+
service: "api",
|
|
93
|
+
type: "int",
|
|
94
|
+
path: "__testkit__/health/http-failure.int.testkit.ts",
|
|
95
|
+
error: "Default runtime thresholds failed: checks(rate==1.0)",
|
|
96
|
+
failureDetails: [
|
|
97
|
+
{
|
|
98
|
+
key: "GET /missing > GET /missing returns 200",
|
|
99
|
+
title: "GET /missing returns 200",
|
|
100
|
+
},
|
|
101
|
+
],
|
|
102
|
+
});
|
|
103
|
+
|
|
104
|
+
expect(matches).toHaveLength(1);
|
|
105
|
+
expect(matches[0].id).toBe("missing-route");
|
|
106
|
+
});
|
|
107
|
+
|
|
62
108
|
it("validates status coverage and filesystem matches", () => {
|
|
63
109
|
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "testkit-known-failures-"));
|
|
64
110
|
tempDirs.push(tempDir);
|
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
import path from "path";
|
|
2
|
+
import { fileURLToPath } from "url";
|
|
3
|
+
|
|
1
4
|
export function determineDefaultRuntimeFailure(result, summary, firstLine) {
|
|
2
5
|
const fatalRuntimeError = extractDefaultRuntimeFatalError(result.stderr || "", firstLine);
|
|
3
6
|
if (fatalRuntimeError) {
|
|
@@ -26,6 +29,22 @@ export function extractDefaultRuntimeFatalError(stderr, firstLine) {
|
|
|
26
29
|
return matched?.[1]?.trim() || firstLine(stderr);
|
|
27
30
|
}
|
|
28
31
|
|
|
32
|
+
export function extractDefaultRuntimeFatalDetail(stderr, firstLine) {
|
|
33
|
+
if (!stderr || !/source=stacktrace/.test(stderr)) return null;
|
|
34
|
+
const message = extractDefaultRuntimeFatalError(stderr, firstLine);
|
|
35
|
+
if (!message) return null;
|
|
36
|
+
|
|
37
|
+
const location = extractFirstLocation(stderr);
|
|
38
|
+
return {
|
|
39
|
+
kind: "runtime-exception",
|
|
40
|
+
key: location ? `${location.path}:${location.line}:${location.column}` : message,
|
|
41
|
+
title: "Uncaught runtime exception",
|
|
42
|
+
message: `Uncaught testkit suite error: ${message}`,
|
|
43
|
+
location,
|
|
44
|
+
stack: sanitizeStack(stderr),
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
|
|
29
48
|
export function extractDefaultRuntimeThresholdFailures(summary) {
|
|
30
49
|
const metrics = summary?.metrics;
|
|
31
50
|
if (!metrics || typeof metrics !== "object") return [];
|
|
@@ -51,3 +70,50 @@ export function sanitizeDefaultRuntimeExitError(exitCode, output, firstLine) {
|
|
|
51
70
|
}
|
|
52
71
|
return `Default runtime failed with exit code ${exitCode}`;
|
|
53
72
|
}
|
|
73
|
+
|
|
74
|
+
function extractFirstLocation(stderr) {
|
|
75
|
+
const locations = [...String(stderr).matchAll(/(file:\/\/[^\s)]+|\/[^\s):]+):(\d+):(\d+)/g)].map((match) => ({
|
|
76
|
+
path: normalizeLocationPath(match[1]),
|
|
77
|
+
line: Number(match[2]),
|
|
78
|
+
column: Number(match[3]),
|
|
79
|
+
}));
|
|
80
|
+
if (locations.length === 0) return null;
|
|
81
|
+
|
|
82
|
+
return (
|
|
83
|
+
locations.find((location) => isPreferredLocation(location.path)) ||
|
|
84
|
+
locations.find((location) => !isRuntimeInternalLocation(location.path)) ||
|
|
85
|
+
locations[0]
|
|
86
|
+
);
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
function normalizeLocationPath(rawPath) {
|
|
90
|
+
if (!rawPath) return rawPath;
|
|
91
|
+
if (rawPath.startsWith("file://")) {
|
|
92
|
+
try {
|
|
93
|
+
return fileURLToPath(rawPath);
|
|
94
|
+
} catch {
|
|
95
|
+
return rawPath;
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
return rawPath;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
function sanitizeStack(stderr) {
|
|
102
|
+
return String(stderr)
|
|
103
|
+
.split(/\r?\n/)
|
|
104
|
+
.map((line) => line.trim())
|
|
105
|
+
.filter(Boolean)
|
|
106
|
+
.join("\n");
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
function isPreferredLocation(filePath) {
|
|
110
|
+
if (typeof filePath !== "string") return false;
|
|
111
|
+
const normalized = filePath.split(path.sep).join("/");
|
|
112
|
+
return normalized.includes("/.testkit/_bundles/") || normalized.includes("/__testkit__/");
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
function isRuntimeInternalLocation(filePath) {
|
|
116
|
+
if (typeof filePath !== "string") return false;
|
|
117
|
+
const normalized = filePath.split(path.sep).join("/");
|
|
118
|
+
return normalized.includes("/testkit/lib/runtime-src/k6/") || normalized.includes("/vendor/k6");
|
|
119
|
+
}
|
|
@@ -8,7 +8,10 @@ import {
|
|
|
8
8
|
formatFileTimeoutBudgetError,
|
|
9
9
|
} from "../shared/file-timeout.mjs";
|
|
10
10
|
import { persistTaskArtifacts, persistTaskOutputArtifacts } from "./artifacts.mjs";
|
|
11
|
-
import {
|
|
11
|
+
import {
|
|
12
|
+
determineDefaultRuntimeFailure,
|
|
13
|
+
extractDefaultRuntimeFatalDetail,
|
|
14
|
+
} from "./default-runtime-errors.mjs";
|
|
12
15
|
import { collectFailureDetailsFromRuntimeArtifacts } from "./failure-details.mjs";
|
|
13
16
|
import { RUNTIME_ARTIFACT_MARKER } from "../runtime-src/k6/artifacts.js";
|
|
14
17
|
import { readDatabaseUrl } from "./state-io.mjs";
|
|
@@ -147,6 +150,10 @@ export async function runDefaultRuntimeTask(
|
|
|
147
150
|
: null,
|
|
148
151
|
]);
|
|
149
152
|
const failureDetails = collectFailureDetailsFromRuntimeArtifacts(rawRuntimeArtifacts);
|
|
153
|
+
const fatalRuntimeDetail = extractDefaultRuntimeFatalDetail(result.stderr || "", getFirstLine);
|
|
154
|
+
if (fatalRuntimeDetail && !failureDetails.some((detail) => detail?.kind === "runtime-exception")) {
|
|
155
|
+
failureDetails.unshift(fatalRuntimeDetail);
|
|
156
|
+
}
|
|
150
157
|
const runtimeError = timedOut
|
|
151
158
|
? formatFileTimeoutBudgetError(fileTimeoutSeconds)
|
|
152
159
|
: determineDefaultRuntimeFailure(result, summary, getFirstLine);
|
|
@@ -28,6 +28,24 @@ export function normalizeFailureDetail(detail) {
|
|
|
28
28
|
const message = normalizeNonEmptyString(detail.message);
|
|
29
29
|
if (message) normalized.message = message;
|
|
30
30
|
|
|
31
|
+
if (detail.expected !== undefined) normalized.expected = cloneJsonValue(detail.expected);
|
|
32
|
+
if (detail.actual !== undefined) normalized.actual = cloneJsonValue(detail.actual);
|
|
33
|
+
|
|
34
|
+
const traceId = normalizeNonEmptyString(detail.traceId);
|
|
35
|
+
if (traceId) normalized.traceId = traceId;
|
|
36
|
+
|
|
37
|
+
const request = normalizeRecord(detail.request);
|
|
38
|
+
if (request) normalized.request = request;
|
|
39
|
+
|
|
40
|
+
const response = normalizeRecord(detail.response);
|
|
41
|
+
if (response) normalized.response = response;
|
|
42
|
+
|
|
43
|
+
const location = normalizeRecord(detail.location);
|
|
44
|
+
if (location) normalized.location = location;
|
|
45
|
+
|
|
46
|
+
const stack = normalizeNonEmptyString(detail.stack);
|
|
47
|
+
if (stack) normalized.stack = stack;
|
|
48
|
+
|
|
31
49
|
return normalized;
|
|
32
50
|
}
|
|
33
51
|
|
|
@@ -89,3 +107,16 @@ function normalizePositiveInteger(value) {
|
|
|
89
107
|
if (!Number.isInteger(value) || value <= 0) return null;
|
|
90
108
|
return value;
|
|
91
109
|
}
|
|
110
|
+
|
|
111
|
+
function normalizeRecord(value) {
|
|
112
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return null;
|
|
113
|
+
return cloneJsonValue(value);
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
function cloneJsonValue(value) {
|
|
117
|
+
try {
|
|
118
|
+
return JSON.parse(JSON.stringify(value));
|
|
119
|
+
} catch {
|
|
120
|
+
return String(value);
|
|
121
|
+
}
|
|
122
|
+
}
|
|
@@ -60,4 +60,55 @@ describe("runner failure details", () => {
|
|
|
60
60
|
},
|
|
61
61
|
]);
|
|
62
62
|
});
|
|
63
|
+
|
|
64
|
+
it("preserves rich assertion metadata", () => {
|
|
65
|
+
expect(
|
|
66
|
+
mergeFailureDetails([
|
|
67
|
+
{
|
|
68
|
+
kind: "http-assertion",
|
|
69
|
+
key: "GET /health > status is 200",
|
|
70
|
+
title: "status is 200",
|
|
71
|
+
expected: 200,
|
|
72
|
+
actual: 404,
|
|
73
|
+
request: {
|
|
74
|
+
method: "GET",
|
|
75
|
+
path: "/health",
|
|
76
|
+
requestId: "req-1",
|
|
77
|
+
},
|
|
78
|
+
response: {
|
|
79
|
+
status: 404,
|
|
80
|
+
bodyPreview: '{"error":"nope"}',
|
|
81
|
+
},
|
|
82
|
+
location: {
|
|
83
|
+
path: "/tmp/example.ts",
|
|
84
|
+
line: 12,
|
|
85
|
+
column: 4,
|
|
86
|
+
},
|
|
87
|
+
},
|
|
88
|
+
])
|
|
89
|
+
).toEqual([
|
|
90
|
+
{
|
|
91
|
+
kind: "http-assertion",
|
|
92
|
+
key: "GET /health > status is 200",
|
|
93
|
+
title: "status is 200",
|
|
94
|
+
count: 1,
|
|
95
|
+
expected: 200,
|
|
96
|
+
actual: 404,
|
|
97
|
+
request: {
|
|
98
|
+
method: "GET",
|
|
99
|
+
path: "/health",
|
|
100
|
+
requestId: "req-1",
|
|
101
|
+
},
|
|
102
|
+
response: {
|
|
103
|
+
status: 404,
|
|
104
|
+
bodyPreview: '{"error":"nope"}',
|
|
105
|
+
},
|
|
106
|
+
location: {
|
|
107
|
+
path: "/tmp/example.ts",
|
|
108
|
+
line: 12,
|
|
109
|
+
column: 4,
|
|
110
|
+
},
|
|
111
|
+
},
|
|
112
|
+
]);
|
|
113
|
+
});
|
|
63
114
|
});
|