@elench/testkit 0.1.52 → 0.1.54
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -0
- package/bin/testkit.mjs +4 -6
- package/lib/cli/command-helpers.mjs +170 -0
- package/lib/cli/commands/artifacts.mjs +45 -0
- package/lib/cli/commands/cleanup.mjs +15 -0
- package/lib/cli/commands/db/snapshot/capture.mjs +22 -0
- package/lib/cli/commands/destroy.mjs +15 -0
- package/lib/cli/commands/known-failures/render.mjs +19 -0
- package/lib/cli/commands/known-failures/validate.mjs +20 -0
- package/lib/cli/commands/logs.mjs +47 -0
- package/lib/cli/commands/run.mjs +23 -0
- package/lib/cli/commands/show.mjs +47 -0
- package/lib/cli/commands/status.mjs +15 -0
- package/lib/cli/commands/watch.mjs +23 -0
- package/lib/cli/entrypoint.mjs +83 -0
- package/lib/cli/index.mjs +6 -116
- package/lib/cli/presentation/code-frames.mjs +57 -0
- package/lib/cli/presentation/code-frames.test.mjs +71 -0
- package/lib/cli/presentation/colors.mjs +29 -0
- package/lib/cli/presentation/run-reporter.mjs +100 -0
- package/lib/cli/tui/watch-app.mjs +104 -0
- package/lib/cli/viewer.mjs +268 -0
- package/lib/known-failures/index.mjs +1 -1
- package/lib/known-failures/index.test.mjs +46 -0
- package/lib/runner/artifacts.mjs +35 -0
- package/lib/runner/default-runtime-errors.mjs +66 -0
- package/lib/runner/default-runtime-runner.mjs +52 -11
- package/lib/runner/failure-details.mjs +31 -0
- package/lib/runner/failure-details.test.mjs +51 -0
- package/lib/runner/formatting.mjs +207 -0
- package/lib/runner/formatting.test.mjs +81 -6
- package/lib/runner/logs.mjs +89 -0
- package/lib/runner/orchestrator.mjs +51 -20
- package/lib/runner/playwright-runner.mjs +15 -7
- package/lib/runner/processes.mjs +9 -11
- package/lib/runner/reporting.mjs +5 -1
- package/lib/runner/reporting.test.mjs +4 -1
- package/lib/runner/runtime-contexts.mjs +7 -3
- package/lib/runner/runtime-manager.mjs +8 -2
- package/lib/runner/runtime-preparation.mjs +9 -4
- package/lib/runner/services.mjs +25 -8
- package/lib/runner/template-steps.mjs +4 -3
- package/lib/runner/triage.mjs +67 -0
- package/lib/runner/worker-loop.mjs +8 -7
- package/lib/runtime/index.d.ts +60 -0
- package/lib/runtime/index.mjs +12 -0
- package/lib/runtime-src/k6/checks.js +45 -12
- package/lib/runtime-src/k6/http-assertions.js +214 -0
- package/lib/runtime-src/k6/http.js +261 -13
- package/lib/runtime-src/k6/suite.js +46 -1
- package/lib/toolchains/index.mjs +6 -3
- package/package.json +13 -3
|
@@ -0,0 +1,268 @@
|
|
|
1
|
+
import fs from "fs";
|
|
2
|
+
import path from "path";
|
|
3
|
+
import { formatDuration } from "../runner/formatting.mjs";
|
|
4
|
+
import { findLogSliceByRequestId, readLogTail } from "../runner/logs.mjs";
|
|
5
|
+
import {
|
|
6
|
+
findFailureLocation,
|
|
7
|
+
formatLocation,
|
|
8
|
+
renderCodeFrame,
|
|
9
|
+
} from "./presentation/code-frames.mjs";
|
|
10
|
+
|
|
11
|
+
export function loadLatestRunArtifact(productDir) {
|
|
12
|
+
const artifactPath = path.join(productDir, ".testkit", "results", "latest.json");
|
|
13
|
+
if (!fs.existsSync(artifactPath)) {
|
|
14
|
+
throw new Error(`No run artifact found at ${path.relative(productDir, artifactPath)}`);
|
|
15
|
+
}
|
|
16
|
+
return JSON.parse(fs.readFileSync(artifactPath, "utf8"));
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export function resolveFileSubject(runArtifact, selector = null, serviceFilter = null) {
|
|
20
|
+
const files = collectFiles(runArtifact, serviceFilter);
|
|
21
|
+
if (files.length === 0) {
|
|
22
|
+
throw new Error("No file results were found in the latest run artifact.");
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
if (!selector) {
|
|
26
|
+
return files.find((entry) => entry.file.status === "failed") || files[0];
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
const normalizedSelector = normalizePath(selector);
|
|
30
|
+
const exact = files.filter((entry) => entry.file.path === normalizedSelector);
|
|
31
|
+
if (exact.length === 1) return exact[0];
|
|
32
|
+
if (exact.length > 1) {
|
|
33
|
+
throw new Error(`Multiple files matched "${selector}". Re-run with --service to disambiguate.`);
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
const suffixMatches = files.filter((entry) => entry.file.path.endsWith(normalizedSelector));
|
|
37
|
+
if (suffixMatches.length === 1) return suffixMatches[0];
|
|
38
|
+
if (suffixMatches.length > 1) {
|
|
39
|
+
throw new Error(`Multiple files matched "${selector}". Re-run with --service to disambiguate.`);
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
throw new Error(`No file matched "${selector}".`);
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export function collectArtifactEntries(productDir, runArtifact, selector = null, serviceFilter = null) {
|
|
46
|
+
const files = selector
|
|
47
|
+
? [resolveFileSubject(runArtifact, selector, serviceFilter)]
|
|
48
|
+
: collectFiles(runArtifact, serviceFilter);
|
|
49
|
+
const entries = [];
|
|
50
|
+
|
|
51
|
+
for (const entry of files) {
|
|
52
|
+
for (const artifactRef of entry.file.artifacts || []) {
|
|
53
|
+
const absolutePath = path.join(productDir, artifactRef.path);
|
|
54
|
+
const payload = fs.existsSync(absolutePath)
|
|
55
|
+
? JSON.parse(fs.readFileSync(absolutePath, "utf8"))
|
|
56
|
+
: null;
|
|
57
|
+
entries.push({
|
|
58
|
+
...entry,
|
|
59
|
+
artifactRef,
|
|
60
|
+
absolutePath,
|
|
61
|
+
payload,
|
|
62
|
+
});
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
return entries;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
export function formatFileDetail(productDir, runArtifact, subject, options = {}) {
|
|
70
|
+
const lines = [];
|
|
71
|
+
const failureDetails = subject.file.failureDetails || [];
|
|
72
|
+
const primaryDetail = rankFailureDetails(failureDetails)[0] || null;
|
|
73
|
+
lines.push(`File: ${subject.file.path}`);
|
|
74
|
+
lines.push(`Service: ${subject.service.name}`);
|
|
75
|
+
lines.push(`Suite: ${subject.suite.type}:${subject.suite.name}`);
|
|
76
|
+
lines.push(`Status: ${subject.file.status}`);
|
|
77
|
+
lines.push(`Duration: ${formatDuration(subject.file.durationMs || 0)}`);
|
|
78
|
+
if (subject.file.error) lines.push(`Error: ${subject.file.error}`);
|
|
79
|
+
|
|
80
|
+
if (failureDetails.length > 0) {
|
|
81
|
+
lines.push("");
|
|
82
|
+
lines.push("Failure Details:");
|
|
83
|
+
for (const detail of rankFailureDetails(failureDetails).slice(0, options.failureLimit || 5)) {
|
|
84
|
+
lines.push(` ${detail.title}`);
|
|
85
|
+
if (detail.message) lines.push(` ${detail.message}`);
|
|
86
|
+
const requestLine = formatRequestLine(detail);
|
|
87
|
+
if (requestLine) lines.push(` ${requestLine}`);
|
|
88
|
+
const responseLine = formatResponseLine(detail);
|
|
89
|
+
if (responseLine) lines.push(` ${responseLine}`);
|
|
90
|
+
const location = findFailureLocation(detail, subject.file.error || "");
|
|
91
|
+
if (location) lines.push(` at ${formatLocation(location, productDir)}`);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
const codeFrame = renderCodeFrame(findFailureLocation(primaryDetail, subject.file.error || ""), {
|
|
96
|
+
cwd: productDir,
|
|
97
|
+
});
|
|
98
|
+
if (codeFrame.length > 0) {
|
|
99
|
+
lines.push("");
|
|
100
|
+
lines.push("Code Frame:");
|
|
101
|
+
for (const line of codeFrame) lines.push(` ${line}`);
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
if (subject.file.triage) {
|
|
105
|
+
lines.push("");
|
|
106
|
+
lines.push("Triage:");
|
|
107
|
+
const triageLines = formatTriage(subject.file.triage);
|
|
108
|
+
for (const line of triageLines) lines.push(` ${line}`);
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
const artifacts = collectArtifactEntries(productDir, runArtifact, subject.file.path, subject.service.name);
|
|
112
|
+
if (artifacts.length > 0) {
|
|
113
|
+
lines.push("");
|
|
114
|
+
lines.push("Artifacts:");
|
|
115
|
+
for (const entry of artifacts) {
|
|
116
|
+
lines.push(` ${entry.artifactRef.name}${entry.artifactRef.kind ? ` [${entry.artifactRef.kind}]` : ""}`);
|
|
117
|
+
if (entry.artifactRef.summary) lines.push(` ${entry.artifactRef.summary}`);
|
|
118
|
+
for (const previewLine of formatArtifactPreview(entry.payload, options.previewLength || 6)) {
|
|
119
|
+
lines.push(` ${previewLine}`);
|
|
120
|
+
}
|
|
121
|
+
lines.push(` ${entry.artifactRef.path}`);
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
const logRefs = getServiceLogRefs(runArtifact, subject.service.name);
|
|
126
|
+
if (logRefs.length > 0) {
|
|
127
|
+
lines.push("");
|
|
128
|
+
lines.push("Backend Logs:");
|
|
129
|
+
for (const logRef of logRefs) {
|
|
130
|
+
lines.push(` ${logRef.runtimeLabel}`);
|
|
131
|
+
lines.push(` ${logRef.path}`);
|
|
132
|
+
const logPath = path.join(productDir, logRef.path);
|
|
133
|
+
const requestId = primaryDetail?.request?.requestId || null;
|
|
134
|
+
const tail =
|
|
135
|
+
requestId && findLogSliceByRequestId(logPath, requestId, 2).length > 0
|
|
136
|
+
? findLogSliceByRequestId(logPath, requestId, 2)
|
|
137
|
+
: readLogTail(logPath, options.logTail || 12);
|
|
138
|
+
for (const line of tail.slice(-Math.max(0, options.logTail || 12))) {
|
|
139
|
+
lines.push(` ${line}`);
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
return lines;
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
export function getServiceLogRefs(runArtifact, serviceName) {
|
|
148
|
+
return (runArtifact.logs?.services || []).filter((entry) => entry.serviceName === serviceName);
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
export function formatArtifactPreview(payload, maxLines = 6) {
|
|
152
|
+
if (!payload) return ["artifact payload missing"];
|
|
153
|
+
if (payload.kind === "agentic-query") {
|
|
154
|
+
return formatAgenticArtifact(payload, maxLines);
|
|
155
|
+
}
|
|
156
|
+
if (payload.kind === "testkit.http-traces") {
|
|
157
|
+
return formatHttpTraceArtifact(payload, maxLines);
|
|
158
|
+
}
|
|
159
|
+
if (payload.contentType === "text/plain" && typeof payload.data?.text === "string") {
|
|
160
|
+
return payload.data.text
|
|
161
|
+
.split(/\r?\n/)
|
|
162
|
+
.filter((line) => line.trim().length > 0)
|
|
163
|
+
.slice(0, maxLines);
|
|
164
|
+
}
|
|
165
|
+
const preview = JSON.stringify(payload.data, null, 2)
|
|
166
|
+
.split(/\r?\n/)
|
|
167
|
+
.slice(0, maxLines);
|
|
168
|
+
return preview;
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
function formatAgenticArtifact(payload, maxLines) {
|
|
172
|
+
const artifact = payload.data || {};
|
|
173
|
+
const lines = [];
|
|
174
|
+
if (artifact.query?.text) lines.push(`Query: ${artifact.query.text}`);
|
|
175
|
+
if (artifact.ui?.latestAssistantMessage?.content) {
|
|
176
|
+
lines.push(`Answer: ${String(artifact.ui.latestAssistantMessage.content).replace(/\s+/g, " ").trim()}`);
|
|
177
|
+
}
|
|
178
|
+
if (Array.isArray(artifact.ui?.resultTables) && artifact.ui.resultTables.length > 0) {
|
|
179
|
+
const table = artifact.ui.resultTables[0];
|
|
180
|
+
lines.push(`Table: ${table.rowCount} rows · columns ${table.columns.join(", ")}`);
|
|
181
|
+
}
|
|
182
|
+
return lines.slice(0, maxLines);
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
function formatHttpTraceArtifact(payload, maxLines) {
|
|
186
|
+
const traces = Array.isArray(payload.data?.traces) ? payload.data.traces : [];
|
|
187
|
+
const lines = [];
|
|
188
|
+
for (const trace of traces.slice(0, maxLines)) {
|
|
189
|
+
lines.push(
|
|
190
|
+
`${trace.method} ${trace.path} -> ${trace.response?.status ?? "?"}${trace.requestId ? ` [${trace.requestId}]` : ""}`
|
|
191
|
+
);
|
|
192
|
+
}
|
|
193
|
+
return lines;
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
function rankFailureDetails(details) {
|
|
197
|
+
return [...(Array.isArray(details) ? details : [])].sort((left, right) => {
|
|
198
|
+
return failureDetailRank(left) - failureDetailRank(right) || String(left?.key || "").localeCompare(String(right?.key || ""));
|
|
199
|
+
});
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
function failureDetailRank(detail) {
|
|
203
|
+
if (detail?.kind === "http-assertion") return 1;
|
|
204
|
+
if (detail?.request && detail?.response) return 2;
|
|
205
|
+
if (detail?.location || detail?.stack) return 3;
|
|
206
|
+
if (detail?.message) return 4;
|
|
207
|
+
return 5;
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
function formatRequestLine(detail) {
|
|
211
|
+
const method = detail?.request?.method;
|
|
212
|
+
const path = detail?.request?.path;
|
|
213
|
+
if (!method || !path) return null;
|
|
214
|
+
const requestId = detail?.request?.requestId;
|
|
215
|
+
return requestId ? `request: ${method} ${path} [${requestId}]` : `request: ${method} ${path}`;
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
function formatResponseLine(detail) {
|
|
219
|
+
if (!detail?.response) return null;
|
|
220
|
+
const parts = [`response: ${detail.response.status ?? "?"}`];
|
|
221
|
+
if (detail.response.contentType) parts.push(detail.response.contentType);
|
|
222
|
+
if (detail.response.bodyPreview) parts.push(detail.response.bodyPreview);
|
|
223
|
+
return parts.join(" ");
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
function formatTriage(triage) {
|
|
227
|
+
const lines = [`status: ${triage.status}`];
|
|
228
|
+
if (triage.classifications?.length) {
|
|
229
|
+
lines.push(`classification: ${triage.classifications.join(", ")}`);
|
|
230
|
+
}
|
|
231
|
+
if (triage.availability?.mode) {
|
|
232
|
+
lines.push(
|
|
233
|
+
`validation: ${triage.availability.mode}${triage.availability.reason ? ` (${triage.availability.reason})` : ""}`
|
|
234
|
+
);
|
|
235
|
+
}
|
|
236
|
+
for (const entry of triage.entries || []) {
|
|
237
|
+
lines.push(`issue: ${entry.issue.repo}#${entry.issue.number}`);
|
|
238
|
+
if (entry.issue.url) lines.push(`url: ${entry.issue.url}`);
|
|
239
|
+
if (entry.github?.state) {
|
|
240
|
+
lines.push(`github state: ${entry.github.state}${entry.github.cached ? " (cache)" : ""}`);
|
|
241
|
+
}
|
|
242
|
+
if (entry.validationStatus) lines.push(`validation status: ${entry.validationStatus}`);
|
|
243
|
+
if (entry.findings?.length) {
|
|
244
|
+
for (const finding of entry.findings.slice(0, 3)) {
|
|
245
|
+
lines.push(`finding: ${finding.message}`);
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
break;
|
|
249
|
+
}
|
|
250
|
+
return lines;
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
function collectFiles(runArtifact, serviceFilter = null) {
|
|
254
|
+
const files = [];
|
|
255
|
+
for (const service of runArtifact.services || []) {
|
|
256
|
+
if (serviceFilter && service.name !== serviceFilter) continue;
|
|
257
|
+
for (const suite of service.suites || []) {
|
|
258
|
+
for (const file of suite.files || []) {
|
|
259
|
+
files.push({ service, suite, file });
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
return files;
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
function normalizePath(filePath) {
|
|
267
|
+
return String(filePath).split(path.sep).join("/");
|
|
268
|
+
}
|
|
@@ -194,7 +194,7 @@ export function matchesKnownFailureMatch(match, fileSummary) {
|
|
|
194
194
|
if (match.path !== fileSummary.path) return false;
|
|
195
195
|
if (match.failureKey) {
|
|
196
196
|
const failureKeys = Array.isArray(fileSummary.failureDetails)
|
|
197
|
-
? fileSummary.failureDetails.
|
|
197
|
+
? fileSummary.failureDetails.flatMap((detail) => [detail?.key, detail?.title].filter(Boolean))
|
|
198
198
|
: [];
|
|
199
199
|
if (!failureKeys.includes(match.failureKey)) return false;
|
|
200
200
|
}
|
|
@@ -59,6 +59,52 @@ describe("known failures core", () => {
|
|
|
59
59
|
expect(matches[0].id).toBe("bad-message");
|
|
60
60
|
});
|
|
61
61
|
|
|
62
|
+
it("matches failureKey against a detail title when the key becomes richer", () => {
|
|
63
|
+
const document = normalizeKnownFailuresDocument({
|
|
64
|
+
schemaVersion: 1,
|
|
65
|
+
entries: [
|
|
66
|
+
{
|
|
67
|
+
id: "missing-route",
|
|
68
|
+
title: "Missing route bug",
|
|
69
|
+
classification: "product_bug",
|
|
70
|
+
state: "open",
|
|
71
|
+
issue: {
|
|
72
|
+
repo: "acme/repo",
|
|
73
|
+
number: 13,
|
|
74
|
+
url: "https://github.com/acme/repo/issues/13",
|
|
75
|
+
},
|
|
76
|
+
description: "Wrong status code",
|
|
77
|
+
whyFailing: "The route returns 404",
|
|
78
|
+
lastReviewedAt: "2026-04-28",
|
|
79
|
+
matches: [
|
|
80
|
+
{
|
|
81
|
+
service: "api",
|
|
82
|
+
type: "int",
|
|
83
|
+
path: "__testkit__/health/http-failure.int.testkit.ts",
|
|
84
|
+
failureKey: "GET /missing returns 200",
|
|
85
|
+
},
|
|
86
|
+
],
|
|
87
|
+
},
|
|
88
|
+
],
|
|
89
|
+
});
|
|
90
|
+
|
|
91
|
+
const matches = findMatchingKnownFailureEntries(document, {
|
|
92
|
+
service: "api",
|
|
93
|
+
type: "int",
|
|
94
|
+
path: "__testkit__/health/http-failure.int.testkit.ts",
|
|
95
|
+
error: "Default runtime thresholds failed: checks(rate==1.0)",
|
|
96
|
+
failureDetails: [
|
|
97
|
+
{
|
|
98
|
+
key: "GET /missing > GET /missing returns 200",
|
|
99
|
+
title: "GET /missing returns 200",
|
|
100
|
+
},
|
|
101
|
+
],
|
|
102
|
+
});
|
|
103
|
+
|
|
104
|
+
expect(matches).toHaveLength(1);
|
|
105
|
+
expect(matches[0].id).toBe("missing-route");
|
|
106
|
+
});
|
|
107
|
+
|
|
62
108
|
it("validates status coverage and filesystem matches", () => {
|
|
63
109
|
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "testkit-known-failures-"));
|
|
64
110
|
tempDirs.push(tempDir);
|
package/lib/runner/artifacts.mjs
CHANGED
|
@@ -8,6 +8,7 @@ import {
|
|
|
8
8
|
|
|
9
9
|
const TIMINGS_FILENAME = "timings.json";
|
|
10
10
|
const RESULT_ARTIFACTS_DIRNAME = "artifacts";
|
|
11
|
+
const RESULT_LOGS_DIRNAME = "logs";
|
|
11
12
|
|
|
12
13
|
export function writeRunArtifact(productDir, artifact) {
|
|
13
14
|
const resultsDir = path.join(productDir, ".testkit", "results");
|
|
@@ -27,6 +28,10 @@ export function resetResultArtifacts(productDir) {
|
|
|
27
28
|
recursive: true,
|
|
28
29
|
force: true,
|
|
29
30
|
});
|
|
31
|
+
fs.rmSync(path.join(productDir, ".testkit", "results", RESULT_LOGS_DIRNAME), {
|
|
32
|
+
recursive: true,
|
|
33
|
+
force: true,
|
|
34
|
+
});
|
|
30
35
|
}
|
|
31
36
|
|
|
32
37
|
export function persistTaskArtifacts(productDir, task, emittedArtifacts) {
|
|
@@ -81,6 +86,27 @@ export function persistTaskArtifacts(productDir, task, emittedArtifacts) {
|
|
|
81
86
|
});
|
|
82
87
|
}
|
|
83
88
|
|
|
89
|
+
export function persistTaskOutputArtifacts(productDir, task, outputs) {
|
|
90
|
+
if (!Array.isArray(outputs) || outputs.length === 0) return [];
|
|
91
|
+
return persistTaskArtifacts(
|
|
92
|
+
productDir,
|
|
93
|
+
task,
|
|
94
|
+
outputs
|
|
95
|
+
.filter((entry) => typeof entry?.text === "string" && entry.text.trim().length > 0)
|
|
96
|
+
.map((entry) => ({
|
|
97
|
+
name: entry.name || "task-output",
|
|
98
|
+
kind: entry.kind || "runtime.output",
|
|
99
|
+
summary: entry.summary || summarizeOutput(entry.text),
|
|
100
|
+
contentType: entry.contentType || "text/plain",
|
|
101
|
+
emittedAt: entry.emittedAt || new Date().toISOString(),
|
|
102
|
+
data: {
|
|
103
|
+
stream: entry.stream || null,
|
|
104
|
+
text: entry.text,
|
|
105
|
+
},
|
|
106
|
+
}))
|
|
107
|
+
);
|
|
108
|
+
}
|
|
109
|
+
|
|
84
110
|
export function loadTimings(productDir) {
|
|
85
111
|
const filePath = path.join(productDir, ".testkit", TIMINGS_FILENAME);
|
|
86
112
|
if (!fs.existsSync(filePath)) {
|
|
@@ -113,3 +139,12 @@ function sanitizePathSegment(value) {
|
|
|
113
139
|
function normalizePath(filePath) {
|
|
114
140
|
return filePath.split(path.sep).join("/");
|
|
115
141
|
}
|
|
142
|
+
|
|
143
|
+
function summarizeOutput(text) {
|
|
144
|
+
const firstLine = String(text)
|
|
145
|
+
.split(/\r?\n/)
|
|
146
|
+
.map((line) => line.trim())
|
|
147
|
+
.find(Boolean);
|
|
148
|
+
if (!firstLine) return "captured task output";
|
|
149
|
+
return firstLine.slice(0, 120);
|
|
150
|
+
}
|
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
import path from "path";
|
|
2
|
+
import { fileURLToPath } from "url";
|
|
3
|
+
|
|
1
4
|
export function determineDefaultRuntimeFailure(result, summary, firstLine) {
|
|
2
5
|
const fatalRuntimeError = extractDefaultRuntimeFatalError(result.stderr || "", firstLine);
|
|
3
6
|
if (fatalRuntimeError) {
|
|
@@ -26,6 +29,22 @@ export function extractDefaultRuntimeFatalError(stderr, firstLine) {
|
|
|
26
29
|
return matched?.[1]?.trim() || firstLine(stderr);
|
|
27
30
|
}
|
|
28
31
|
|
|
32
|
+
export function extractDefaultRuntimeFatalDetail(stderr, firstLine) {
|
|
33
|
+
if (!stderr || !/source=stacktrace/.test(stderr)) return null;
|
|
34
|
+
const message = extractDefaultRuntimeFatalError(stderr, firstLine);
|
|
35
|
+
if (!message) return null;
|
|
36
|
+
|
|
37
|
+
const location = extractFirstLocation(stderr);
|
|
38
|
+
return {
|
|
39
|
+
kind: "runtime-exception",
|
|
40
|
+
key: location ? `${location.path}:${location.line}:${location.column}` : message,
|
|
41
|
+
title: "Uncaught runtime exception",
|
|
42
|
+
message: `Uncaught testkit suite error: ${message}`,
|
|
43
|
+
location,
|
|
44
|
+
stack: sanitizeStack(stderr),
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
|
|
29
48
|
export function extractDefaultRuntimeThresholdFailures(summary) {
|
|
30
49
|
const metrics = summary?.metrics;
|
|
31
50
|
if (!metrics || typeof metrics !== "object") return [];
|
|
@@ -51,3 +70,50 @@ export function sanitizeDefaultRuntimeExitError(exitCode, output, firstLine) {
|
|
|
51
70
|
}
|
|
52
71
|
return `Default runtime failed with exit code ${exitCode}`;
|
|
53
72
|
}
|
|
73
|
+
|
|
74
|
+
function extractFirstLocation(stderr) {
|
|
75
|
+
const locations = [...String(stderr).matchAll(/(file:\/\/[^\s)]+|\/[^\s):]+):(\d+):(\d+)/g)].map((match) => ({
|
|
76
|
+
path: normalizeLocationPath(match[1]),
|
|
77
|
+
line: Number(match[2]),
|
|
78
|
+
column: Number(match[3]),
|
|
79
|
+
}));
|
|
80
|
+
if (locations.length === 0) return null;
|
|
81
|
+
|
|
82
|
+
return (
|
|
83
|
+
locations.find((location) => isPreferredLocation(location.path)) ||
|
|
84
|
+
locations.find((location) => !isRuntimeInternalLocation(location.path)) ||
|
|
85
|
+
locations[0]
|
|
86
|
+
);
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
function normalizeLocationPath(rawPath) {
|
|
90
|
+
if (!rawPath) return rawPath;
|
|
91
|
+
if (rawPath.startsWith("file://")) {
|
|
92
|
+
try {
|
|
93
|
+
return fileURLToPath(rawPath);
|
|
94
|
+
} catch {
|
|
95
|
+
return rawPath;
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
return rawPath;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
function sanitizeStack(stderr) {
|
|
102
|
+
return String(stderr)
|
|
103
|
+
.split(/\r?\n/)
|
|
104
|
+
.map((line) => line.trim())
|
|
105
|
+
.filter(Boolean)
|
|
106
|
+
.join("\n");
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
function isPreferredLocation(filePath) {
|
|
110
|
+
if (typeof filePath !== "string") return false;
|
|
111
|
+
const normalized = filePath.split(path.sep).join("/");
|
|
112
|
+
return normalized.includes("/.testkit/_bundles/") || normalized.includes("/__testkit__/");
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
function isRuntimeInternalLocation(filePath) {
|
|
116
|
+
if (typeof filePath !== "string") return false;
|
|
117
|
+
const normalized = filePath.split(path.sep).join("/");
|
|
118
|
+
return normalized.includes("/testkit/lib/runtime-src/k6/") || normalized.includes("/vendor/k6");
|
|
119
|
+
}
|
|
@@ -7,15 +7,18 @@ import {
|
|
|
7
7
|
buildFileTimeoutEnv,
|
|
8
8
|
formatFileTimeoutBudgetError,
|
|
9
9
|
} from "../shared/file-timeout.mjs";
|
|
10
|
-
import { persistTaskArtifacts } from "./artifacts.mjs";
|
|
11
|
-
import {
|
|
10
|
+
import { persistTaskArtifacts, persistTaskOutputArtifacts } from "./artifacts.mjs";
|
|
11
|
+
import {
|
|
12
|
+
determineDefaultRuntimeFailure,
|
|
13
|
+
extractDefaultRuntimeFatalDetail,
|
|
14
|
+
} from "./default-runtime-errors.mjs";
|
|
12
15
|
import { collectFailureDetailsFromRuntimeArtifacts } from "./failure-details.mjs";
|
|
13
16
|
import { RUNTIME_ARTIFACT_MARKER } from "../runtime-src/k6/artifacts.js";
|
|
14
17
|
import { readDatabaseUrl } from "./state-io.mjs";
|
|
15
18
|
import { buildTaskExecutionEnv } from "./template.mjs";
|
|
16
19
|
import { killChildProcess } from "./processes.mjs";
|
|
17
20
|
|
|
18
|
-
export async function runHttpK6Task(targetConfig, task, lifecycle, lease) {
|
|
21
|
+
export async function runHttpK6Task(targetConfig, task, lifecycle, lease, reporter = null) {
|
|
19
22
|
const baseUrl = targetConfig.testkit.local?.baseUrl;
|
|
20
23
|
if (!baseUrl) {
|
|
21
24
|
throw new Error(`Service "${targetConfig.name}" requires local.baseUrl for HTTP suites`);
|
|
@@ -34,11 +37,12 @@ export async function runHttpK6Task(targetConfig, task, lifecycle, lease) {
|
|
|
34
37
|
task,
|
|
35
38
|
lease,
|
|
36
39
|
["run", "--address", "127.0.0.1:0", "-e", `BASE_URL=${baseUrl}`, bundledFile],
|
|
37
|
-
lifecycle
|
|
40
|
+
lifecycle,
|
|
41
|
+
reporter
|
|
38
42
|
);
|
|
39
43
|
}
|
|
40
44
|
|
|
41
|
-
export async function runDalTask(targetConfig, task, lifecycle, lease) {
|
|
45
|
+
export async function runDalTask(targetConfig, task, lifecycle, lease, reporter = null) {
|
|
42
46
|
const databaseUrl = readDatabaseUrl(targetConfig.stateDir);
|
|
43
47
|
if (!databaseUrl) {
|
|
44
48
|
throw new Error(`Service "${targetConfig.name}" requires a database for DAL suites`);
|
|
@@ -57,12 +61,27 @@ export async function runDalTask(targetConfig, task, lifecycle, lease) {
|
|
|
57
61
|
task,
|
|
58
62
|
lease,
|
|
59
63
|
["run", "--address", "127.0.0.1:0", "-e", `DATABASE_URL=${databaseUrl}`, bundledFile],
|
|
60
|
-
lifecycle
|
|
64
|
+
lifecycle,
|
|
65
|
+
reporter
|
|
61
66
|
);
|
|
62
67
|
}
|
|
63
68
|
|
|
64
|
-
export async function runDefaultRuntimeTask(
|
|
69
|
+
export async function runDefaultRuntimeTask(
|
|
70
|
+
targetConfig,
|
|
71
|
+
task,
|
|
72
|
+
lease,
|
|
73
|
+
args,
|
|
74
|
+
lifecycle,
|
|
75
|
+
reporterOrFirstLine,
|
|
76
|
+
maybeFirstLine
|
|
77
|
+
) {
|
|
65
78
|
const k6Binary = resolveK6Binary();
|
|
79
|
+
const reporter =
|
|
80
|
+
reporterOrFirstLine && typeof reporterOrFirstLine === "object"
|
|
81
|
+
? reporterOrFirstLine
|
|
82
|
+
: null;
|
|
83
|
+
const firstLine =
|
|
84
|
+
typeof reporterOrFirstLine === "function" ? reporterOrFirstLine : maybeFirstLine;
|
|
66
85
|
const getFirstLine = firstLine || defaultFirstLine;
|
|
67
86
|
const summaryFile = buildDefaultRuntimeSummaryPath(lease, task);
|
|
68
87
|
fs.mkdirSync(path.dirname(summaryFile), { recursive: true });
|
|
@@ -91,7 +110,7 @@ export async function runDefaultRuntimeTask(targetConfig, task, lease, args, lif
|
|
|
91
110
|
if (subprocess.pid) interruptSubprocess();
|
|
92
111
|
else subprocess.once?.("spawn", interruptSubprocess);
|
|
93
112
|
}
|
|
94
|
-
|
|
113
|
+
reporter?.taskStarted?.(task, targetConfig);
|
|
95
114
|
let result;
|
|
96
115
|
let timedOut;
|
|
97
116
|
try {
|
|
@@ -102,8 +121,6 @@ export async function runDefaultRuntimeTask(targetConfig, task, lease, args, lif
|
|
|
102
121
|
|
|
103
122
|
const stdout = parseDefaultRuntimeOutput(result.stdout || "");
|
|
104
123
|
const stderr = parseDefaultRuntimeOutput(result.stderr || "");
|
|
105
|
-
if (stdout.visibleOutput) process.stdout.write(stdout.visibleOutput);
|
|
106
|
-
if (stderr.visibleOutput) process.stderr.write(stderr.visibleOutput);
|
|
107
124
|
|
|
108
125
|
const summary = readDefaultRuntimeSummary(summaryFile);
|
|
109
126
|
const rawRuntimeArtifacts = [...stdout.artifacts, ...stderr.artifacts];
|
|
@@ -112,7 +129,31 @@ export async function runDefaultRuntimeTask(targetConfig, task, lease, args, lif
|
|
|
112
129
|
task,
|
|
113
130
|
rawRuntimeArtifacts
|
|
114
131
|
);
|
|
132
|
+
const outputArtifacts = persistTaskOutputArtifacts(targetConfig.productDir, task, [
|
|
133
|
+
stdout.visibleOutput
|
|
134
|
+
? {
|
|
135
|
+
name: "default-runtime-stdout",
|
|
136
|
+
kind: "runtime.output",
|
|
137
|
+
summary: defaultFirstLine(stdout.visibleOutput) || "captured stdout",
|
|
138
|
+
stream: "stdout",
|
|
139
|
+
text: stdout.visibleOutput,
|
|
140
|
+
}
|
|
141
|
+
: null,
|
|
142
|
+
stderr.visibleOutput
|
|
143
|
+
? {
|
|
144
|
+
name: "default-runtime-stderr",
|
|
145
|
+
kind: "runtime.output",
|
|
146
|
+
summary: defaultFirstLine(stderr.visibleOutput) || "captured stderr",
|
|
147
|
+
stream: "stderr",
|
|
148
|
+
text: stderr.visibleOutput,
|
|
149
|
+
}
|
|
150
|
+
: null,
|
|
151
|
+
]);
|
|
115
152
|
const failureDetails = collectFailureDetailsFromRuntimeArtifacts(rawRuntimeArtifacts);
|
|
153
|
+
const fatalRuntimeDetail = extractDefaultRuntimeFatalDetail(result.stderr || "", getFirstLine);
|
|
154
|
+
if (fatalRuntimeDetail && !failureDetails.some((detail) => detail?.kind === "runtime-exception")) {
|
|
155
|
+
failureDetails.unshift(fatalRuntimeDetail);
|
|
156
|
+
}
|
|
116
157
|
const runtimeError = timedOut
|
|
117
158
|
? formatFileTimeoutBudgetError(fileTimeoutSeconds)
|
|
118
159
|
: determineDefaultRuntimeFailure(result, summary, getFirstLine);
|
|
@@ -125,7 +166,7 @@ export async function runDefaultRuntimeTask(targetConfig, task, lease, args, lif
|
|
|
125
166
|
durationMs: finishedAt - startedAt,
|
|
126
167
|
startedAt,
|
|
127
168
|
finishedAt,
|
|
128
|
-
artifacts: runtimeArtifacts,
|
|
169
|
+
artifacts: [...runtimeArtifacts, ...outputArtifacts],
|
|
129
170
|
failureDetails,
|
|
130
171
|
};
|
|
131
172
|
}
|
|
@@ -28,6 +28,24 @@ export function normalizeFailureDetail(detail) {
|
|
|
28
28
|
const message = normalizeNonEmptyString(detail.message);
|
|
29
29
|
if (message) normalized.message = message;
|
|
30
30
|
|
|
31
|
+
if (detail.expected !== undefined) normalized.expected = cloneJsonValue(detail.expected);
|
|
32
|
+
if (detail.actual !== undefined) normalized.actual = cloneJsonValue(detail.actual);
|
|
33
|
+
|
|
34
|
+
const traceId = normalizeNonEmptyString(detail.traceId);
|
|
35
|
+
if (traceId) normalized.traceId = traceId;
|
|
36
|
+
|
|
37
|
+
const request = normalizeRecord(detail.request);
|
|
38
|
+
if (request) normalized.request = request;
|
|
39
|
+
|
|
40
|
+
const response = normalizeRecord(detail.response);
|
|
41
|
+
if (response) normalized.response = response;
|
|
42
|
+
|
|
43
|
+
const location = normalizeRecord(detail.location);
|
|
44
|
+
if (location) normalized.location = location;
|
|
45
|
+
|
|
46
|
+
const stack = normalizeNonEmptyString(detail.stack);
|
|
47
|
+
if (stack) normalized.stack = stack;
|
|
48
|
+
|
|
31
49
|
return normalized;
|
|
32
50
|
}
|
|
33
51
|
|
|
@@ -89,3 +107,16 @@ function normalizePositiveInteger(value) {
|
|
|
89
107
|
if (!Number.isInteger(value) || value <= 0) return null;
|
|
90
108
|
return value;
|
|
91
109
|
}
|
|
110
|
+
|
|
111
|
+
function normalizeRecord(value) {
|
|
112
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return null;
|
|
113
|
+
return cloneJsonValue(value);
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
function cloneJsonValue(value) {
|
|
117
|
+
try {
|
|
118
|
+
return JSON.parse(JSON.stringify(value));
|
|
119
|
+
} catch {
|
|
120
|
+
return String(value);
|
|
121
|
+
}
|
|
122
|
+
}
|
|
@@ -60,4 +60,55 @@ describe("runner failure details", () => {
|
|
|
60
60
|
},
|
|
61
61
|
]);
|
|
62
62
|
});
|
|
63
|
+
|
|
64
|
+
it("preserves rich assertion metadata", () => {
|
|
65
|
+
expect(
|
|
66
|
+
mergeFailureDetails([
|
|
67
|
+
{
|
|
68
|
+
kind: "http-assertion",
|
|
69
|
+
key: "GET /health > status is 200",
|
|
70
|
+
title: "status is 200",
|
|
71
|
+
expected: 200,
|
|
72
|
+
actual: 404,
|
|
73
|
+
request: {
|
|
74
|
+
method: "GET",
|
|
75
|
+
path: "/health",
|
|
76
|
+
requestId: "req-1",
|
|
77
|
+
},
|
|
78
|
+
response: {
|
|
79
|
+
status: 404,
|
|
80
|
+
bodyPreview: '{"error":"nope"}',
|
|
81
|
+
},
|
|
82
|
+
location: {
|
|
83
|
+
path: "/tmp/example.ts",
|
|
84
|
+
line: 12,
|
|
85
|
+
column: 4,
|
|
86
|
+
},
|
|
87
|
+
},
|
|
88
|
+
])
|
|
89
|
+
).toEqual([
|
|
90
|
+
{
|
|
91
|
+
kind: "http-assertion",
|
|
92
|
+
key: "GET /health > status is 200",
|
|
93
|
+
title: "status is 200",
|
|
94
|
+
count: 1,
|
|
95
|
+
expected: 200,
|
|
96
|
+
actual: 404,
|
|
97
|
+
request: {
|
|
98
|
+
method: "GET",
|
|
99
|
+
path: "/health",
|
|
100
|
+
requestId: "req-1",
|
|
101
|
+
},
|
|
102
|
+
response: {
|
|
103
|
+
status: 404,
|
|
104
|
+
bodyPreview: '{"error":"nope"}',
|
|
105
|
+
},
|
|
106
|
+
location: {
|
|
107
|
+
path: "/tmp/example.ts",
|
|
108
|
+
line: 12,
|
|
109
|
+
column: 4,
|
|
110
|
+
},
|
|
111
|
+
},
|
|
112
|
+
]);
|
|
113
|
+
});
|
|
63
114
|
});
|