@elench/testkit 0.1.52 → 0.1.54

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/README.md +14 -0
  2. package/bin/testkit.mjs +4 -6
  3. package/lib/cli/command-helpers.mjs +170 -0
  4. package/lib/cli/commands/artifacts.mjs +45 -0
  5. package/lib/cli/commands/cleanup.mjs +15 -0
  6. package/lib/cli/commands/db/snapshot/capture.mjs +22 -0
  7. package/lib/cli/commands/destroy.mjs +15 -0
  8. package/lib/cli/commands/known-failures/render.mjs +19 -0
  9. package/lib/cli/commands/known-failures/validate.mjs +20 -0
  10. package/lib/cli/commands/logs.mjs +47 -0
  11. package/lib/cli/commands/run.mjs +23 -0
  12. package/lib/cli/commands/show.mjs +47 -0
  13. package/lib/cli/commands/status.mjs +15 -0
  14. package/lib/cli/commands/watch.mjs +23 -0
  15. package/lib/cli/entrypoint.mjs +83 -0
  16. package/lib/cli/index.mjs +6 -116
  17. package/lib/cli/presentation/code-frames.mjs +57 -0
  18. package/lib/cli/presentation/code-frames.test.mjs +71 -0
  19. package/lib/cli/presentation/colors.mjs +29 -0
  20. package/lib/cli/presentation/run-reporter.mjs +100 -0
  21. package/lib/cli/tui/watch-app.mjs +104 -0
  22. package/lib/cli/viewer.mjs +268 -0
  23. package/lib/known-failures/index.mjs +1 -1
  24. package/lib/known-failures/index.test.mjs +46 -0
  25. package/lib/runner/artifacts.mjs +35 -0
  26. package/lib/runner/default-runtime-errors.mjs +66 -0
  27. package/lib/runner/default-runtime-runner.mjs +52 -11
  28. package/lib/runner/failure-details.mjs +31 -0
  29. package/lib/runner/failure-details.test.mjs +51 -0
  30. package/lib/runner/formatting.mjs +207 -0
  31. package/lib/runner/formatting.test.mjs +81 -6
  32. package/lib/runner/logs.mjs +89 -0
  33. package/lib/runner/orchestrator.mjs +51 -20
  34. package/lib/runner/playwright-runner.mjs +15 -7
  35. package/lib/runner/processes.mjs +9 -11
  36. package/lib/runner/reporting.mjs +5 -1
  37. package/lib/runner/reporting.test.mjs +4 -1
  38. package/lib/runner/runtime-contexts.mjs +7 -3
  39. package/lib/runner/runtime-manager.mjs +8 -2
  40. package/lib/runner/runtime-preparation.mjs +9 -4
  41. package/lib/runner/services.mjs +25 -8
  42. package/lib/runner/template-steps.mjs +4 -3
  43. package/lib/runner/triage.mjs +67 -0
  44. package/lib/runner/worker-loop.mjs +8 -7
  45. package/lib/runtime/index.d.ts +60 -0
  46. package/lib/runtime/index.mjs +12 -0
  47. package/lib/runtime-src/k6/checks.js +45 -12
  48. package/lib/runtime-src/k6/http-assertions.js +214 -0
  49. package/lib/runtime-src/k6/http.js +261 -13
  50. package/lib/runtime-src/k6/suite.js +46 -1
  51. package/lib/toolchains/index.mjs +6 -3
  52. package/package.json +13 -3
@@ -0,0 +1,268 @@
1
+ import fs from "fs";
2
+ import path from "path";
3
+ import { formatDuration } from "../runner/formatting.mjs";
4
+ import { findLogSliceByRequestId, readLogTail } from "../runner/logs.mjs";
5
+ import {
6
+ findFailureLocation,
7
+ formatLocation,
8
+ renderCodeFrame,
9
+ } from "./presentation/code-frames.mjs";
10
+
11
+ export function loadLatestRunArtifact(productDir) {
12
+ const artifactPath = path.join(productDir, ".testkit", "results", "latest.json");
13
+ if (!fs.existsSync(artifactPath)) {
14
+ throw new Error(`No run artifact found at ${path.relative(productDir, artifactPath)}`);
15
+ }
16
+ return JSON.parse(fs.readFileSync(artifactPath, "utf8"));
17
+ }
18
+
19
+ export function resolveFileSubject(runArtifact, selector = null, serviceFilter = null) {
20
+ const files = collectFiles(runArtifact, serviceFilter);
21
+ if (files.length === 0) {
22
+ throw new Error("No file results were found in the latest run artifact.");
23
+ }
24
+
25
+ if (!selector) {
26
+ return files.find((entry) => entry.file.status === "failed") || files[0];
27
+ }
28
+
29
+ const normalizedSelector = normalizePath(selector);
30
+ const exact = files.filter((entry) => entry.file.path === normalizedSelector);
31
+ if (exact.length === 1) return exact[0];
32
+ if (exact.length > 1) {
33
+ throw new Error(`Multiple files matched "${selector}". Re-run with --service to disambiguate.`);
34
+ }
35
+
36
+ const suffixMatches = files.filter((entry) => entry.file.path.endsWith(normalizedSelector));
37
+ if (suffixMatches.length === 1) return suffixMatches[0];
38
+ if (suffixMatches.length > 1) {
39
+ throw new Error(`Multiple files matched "${selector}". Re-run with --service to disambiguate.`);
40
+ }
41
+
42
+ throw new Error(`No file matched "${selector}".`);
43
+ }
44
+
45
+ export function collectArtifactEntries(productDir, runArtifact, selector = null, serviceFilter = null) {
46
+ const files = selector
47
+ ? [resolveFileSubject(runArtifact, selector, serviceFilter)]
48
+ : collectFiles(runArtifact, serviceFilter);
49
+ const entries = [];
50
+
51
+ for (const entry of files) {
52
+ for (const artifactRef of entry.file.artifacts || []) {
53
+ const absolutePath = path.join(productDir, artifactRef.path);
54
+ const payload = fs.existsSync(absolutePath)
55
+ ? JSON.parse(fs.readFileSync(absolutePath, "utf8"))
56
+ : null;
57
+ entries.push({
58
+ ...entry,
59
+ artifactRef,
60
+ absolutePath,
61
+ payload,
62
+ });
63
+ }
64
+ }
65
+
66
+ return entries;
67
+ }
68
+
69
+ export function formatFileDetail(productDir, runArtifact, subject, options = {}) {
70
+ const lines = [];
71
+ const failureDetails = subject.file.failureDetails || [];
72
+ const primaryDetail = rankFailureDetails(failureDetails)[0] || null;
73
+ lines.push(`File: ${subject.file.path}`);
74
+ lines.push(`Service: ${subject.service.name}`);
75
+ lines.push(`Suite: ${subject.suite.type}:${subject.suite.name}`);
76
+ lines.push(`Status: ${subject.file.status}`);
77
+ lines.push(`Duration: ${formatDuration(subject.file.durationMs || 0)}`);
78
+ if (subject.file.error) lines.push(`Error: ${subject.file.error}`);
79
+
80
+ if (failureDetails.length > 0) {
81
+ lines.push("");
82
+ lines.push("Failure Details:");
83
+ for (const detail of rankFailureDetails(failureDetails).slice(0, options.failureLimit || 5)) {
84
+ lines.push(` ${detail.title}`);
85
+ if (detail.message) lines.push(` ${detail.message}`);
86
+ const requestLine = formatRequestLine(detail);
87
+ if (requestLine) lines.push(` ${requestLine}`);
88
+ const responseLine = formatResponseLine(detail);
89
+ if (responseLine) lines.push(` ${responseLine}`);
90
+ const location = findFailureLocation(detail, subject.file.error || "");
91
+ if (location) lines.push(` at ${formatLocation(location, productDir)}`);
92
+ }
93
+ }
94
+
95
+ const codeFrame = renderCodeFrame(findFailureLocation(primaryDetail, subject.file.error || ""), {
96
+ cwd: productDir,
97
+ });
98
+ if (codeFrame.length > 0) {
99
+ lines.push("");
100
+ lines.push("Code Frame:");
101
+ for (const line of codeFrame) lines.push(` ${line}`);
102
+ }
103
+
104
+ if (subject.file.triage) {
105
+ lines.push("");
106
+ lines.push("Triage:");
107
+ const triageLines = formatTriage(subject.file.triage);
108
+ for (const line of triageLines) lines.push(` ${line}`);
109
+ }
110
+
111
+ const artifacts = collectArtifactEntries(productDir, runArtifact, subject.file.path, subject.service.name);
112
+ if (artifacts.length > 0) {
113
+ lines.push("");
114
+ lines.push("Artifacts:");
115
+ for (const entry of artifacts) {
116
+ lines.push(` ${entry.artifactRef.name}${entry.artifactRef.kind ? ` [${entry.artifactRef.kind}]` : ""}`);
117
+ if (entry.artifactRef.summary) lines.push(` ${entry.artifactRef.summary}`);
118
+ for (const previewLine of formatArtifactPreview(entry.payload, options.previewLength || 6)) {
119
+ lines.push(` ${previewLine}`);
120
+ }
121
+ lines.push(` ${entry.artifactRef.path}`);
122
+ }
123
+ }
124
+
125
+ const logRefs = getServiceLogRefs(runArtifact, subject.service.name);
126
+ if (logRefs.length > 0) {
127
+ lines.push("");
128
+ lines.push("Backend Logs:");
129
+ for (const logRef of logRefs) {
130
+ lines.push(` ${logRef.runtimeLabel}`);
131
+ lines.push(` ${logRef.path}`);
132
+ const logPath = path.join(productDir, logRef.path);
133
+ const requestId = primaryDetail?.request?.requestId || null;
134
+ const tail =
135
+ requestId && findLogSliceByRequestId(logPath, requestId, 2).length > 0
136
+ ? findLogSliceByRequestId(logPath, requestId, 2)
137
+ : readLogTail(logPath, options.logTail || 12);
138
+ for (const line of tail.slice(-Math.max(0, options.logTail || 12))) {
139
+ lines.push(` ${line}`);
140
+ }
141
+ }
142
+ }
143
+
144
+ return lines;
145
+ }
146
+
147
+ export function getServiceLogRefs(runArtifact, serviceName) {
148
+ return (runArtifact.logs?.services || []).filter((entry) => entry.serviceName === serviceName);
149
+ }
150
+
151
+ export function formatArtifactPreview(payload, maxLines = 6) {
152
+ if (!payload) return ["artifact payload missing"];
153
+ if (payload.kind === "agentic-query") {
154
+ return formatAgenticArtifact(payload, maxLines);
155
+ }
156
+ if (payload.kind === "testkit.http-traces") {
157
+ return formatHttpTraceArtifact(payload, maxLines);
158
+ }
159
+ if (payload.contentType === "text/plain" && typeof payload.data?.text === "string") {
160
+ return payload.data.text
161
+ .split(/\r?\n/)
162
+ .filter((line) => line.trim().length > 0)
163
+ .slice(0, maxLines);
164
+ }
165
+ const preview = JSON.stringify(payload.data, null, 2)
166
+ .split(/\r?\n/)
167
+ .slice(0, maxLines);
168
+ return preview;
169
+ }
170
+
171
+ function formatAgenticArtifact(payload, maxLines) {
172
+ const artifact = payload.data || {};
173
+ const lines = [];
174
+ if (artifact.query?.text) lines.push(`Query: ${artifact.query.text}`);
175
+ if (artifact.ui?.latestAssistantMessage?.content) {
176
+ lines.push(`Answer: ${String(artifact.ui.latestAssistantMessage.content).replace(/\s+/g, " ").trim()}`);
177
+ }
178
+ if (Array.isArray(artifact.ui?.resultTables) && artifact.ui.resultTables.length > 0) {
179
+ const table = artifact.ui.resultTables[0];
180
+ lines.push(`Table: ${table.rowCount} rows · columns ${table.columns.join(", ")}`);
181
+ }
182
+ return lines.slice(0, maxLines);
183
+ }
184
+
185
+ function formatHttpTraceArtifact(payload, maxLines) {
186
+ const traces = Array.isArray(payload.data?.traces) ? payload.data.traces : [];
187
+ const lines = [];
188
+ for (const trace of traces.slice(0, maxLines)) {
189
+ lines.push(
190
+ `${trace.method} ${trace.path} -> ${trace.response?.status ?? "?"}${trace.requestId ? ` [${trace.requestId}]` : ""}`
191
+ );
192
+ }
193
+ return lines;
194
+ }
195
+
196
+ function rankFailureDetails(details) {
197
+ return [...(Array.isArray(details) ? details : [])].sort((left, right) => {
198
+ return failureDetailRank(left) - failureDetailRank(right) || String(left?.key || "").localeCompare(String(right?.key || ""));
199
+ });
200
+ }
201
+
202
+ function failureDetailRank(detail) {
203
+ if (detail?.kind === "http-assertion") return 1;
204
+ if (detail?.request && detail?.response) return 2;
205
+ if (detail?.location || detail?.stack) return 3;
206
+ if (detail?.message) return 4;
207
+ return 5;
208
+ }
209
+
210
+ function formatRequestLine(detail) {
211
+ const method = detail?.request?.method;
212
+ const path = detail?.request?.path;
213
+ if (!method || !path) return null;
214
+ const requestId = detail?.request?.requestId;
215
+ return requestId ? `request: ${method} ${path} [${requestId}]` : `request: ${method} ${path}`;
216
+ }
217
+
218
+ function formatResponseLine(detail) {
219
+ if (!detail?.response) return null;
220
+ const parts = [`response: ${detail.response.status ?? "?"}`];
221
+ if (detail.response.contentType) parts.push(detail.response.contentType);
222
+ if (detail.response.bodyPreview) parts.push(detail.response.bodyPreview);
223
+ return parts.join(" ");
224
+ }
225
+
226
+ function formatTriage(triage) {
227
+ const lines = [`status: ${triage.status}`];
228
+ if (triage.classifications?.length) {
229
+ lines.push(`classification: ${triage.classifications.join(", ")}`);
230
+ }
231
+ if (triage.availability?.mode) {
232
+ lines.push(
233
+ `validation: ${triage.availability.mode}${triage.availability.reason ? ` (${triage.availability.reason})` : ""}`
234
+ );
235
+ }
236
+ for (const entry of triage.entries || []) {
237
+ lines.push(`issue: ${entry.issue.repo}#${entry.issue.number}`);
238
+ if (entry.issue.url) lines.push(`url: ${entry.issue.url}`);
239
+ if (entry.github?.state) {
240
+ lines.push(`github state: ${entry.github.state}${entry.github.cached ? " (cache)" : ""}`);
241
+ }
242
+ if (entry.validationStatus) lines.push(`validation status: ${entry.validationStatus}`);
243
+ if (entry.findings?.length) {
244
+ for (const finding of entry.findings.slice(0, 3)) {
245
+ lines.push(`finding: ${finding.message}`);
246
+ }
247
+ }
248
+ break;
249
+ }
250
+ return lines;
251
+ }
252
+
253
+ function collectFiles(runArtifact, serviceFilter = null) {
254
+ const files = [];
255
+ for (const service of runArtifact.services || []) {
256
+ if (serviceFilter && service.name !== serviceFilter) continue;
257
+ for (const suite of service.suites || []) {
258
+ for (const file of suite.files || []) {
259
+ files.push({ service, suite, file });
260
+ }
261
+ }
262
+ }
263
+ return files;
264
+ }
265
+
266
+ function normalizePath(filePath) {
267
+ return String(filePath).split(path.sep).join("/");
268
+ }
@@ -194,7 +194,7 @@ export function matchesKnownFailureMatch(match, fileSummary) {
194
194
  if (match.path !== fileSummary.path) return false;
195
195
  if (match.failureKey) {
196
196
  const failureKeys = Array.isArray(fileSummary.failureDetails)
197
- ? fileSummary.failureDetails.map((detail) => detail?.key)
197
+ ? fileSummary.failureDetails.flatMap((detail) => [detail?.key, detail?.title].filter(Boolean))
198
198
  : [];
199
199
  if (!failureKeys.includes(match.failureKey)) return false;
200
200
  }
@@ -59,6 +59,52 @@ describe("known failures core", () => {
59
59
  expect(matches[0].id).toBe("bad-message");
60
60
  });
61
61
 
62
+ it("matches failureKey against a detail title when the key becomes richer", () => {
63
+ const document = normalizeKnownFailuresDocument({
64
+ schemaVersion: 1,
65
+ entries: [
66
+ {
67
+ id: "missing-route",
68
+ title: "Missing route bug",
69
+ classification: "product_bug",
70
+ state: "open",
71
+ issue: {
72
+ repo: "acme/repo",
73
+ number: 13,
74
+ url: "https://github.com/acme/repo/issues/13",
75
+ },
76
+ description: "Wrong status code",
77
+ whyFailing: "The route returns 404",
78
+ lastReviewedAt: "2026-04-28",
79
+ matches: [
80
+ {
81
+ service: "api",
82
+ type: "int",
83
+ path: "__testkit__/health/http-failure.int.testkit.ts",
84
+ failureKey: "GET /missing returns 200",
85
+ },
86
+ ],
87
+ },
88
+ ],
89
+ });
90
+
91
+ const matches = findMatchingKnownFailureEntries(document, {
92
+ service: "api",
93
+ type: "int",
94
+ path: "__testkit__/health/http-failure.int.testkit.ts",
95
+ error: "Default runtime thresholds failed: checks(rate==1.0)",
96
+ failureDetails: [
97
+ {
98
+ key: "GET /missing > GET /missing returns 200",
99
+ title: "GET /missing returns 200",
100
+ },
101
+ ],
102
+ });
103
+
104
+ expect(matches).toHaveLength(1);
105
+ expect(matches[0].id).toBe("missing-route");
106
+ });
107
+
62
108
  it("validates status coverage and filesystem matches", () => {
63
109
  const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "testkit-known-failures-"));
64
110
  tempDirs.push(tempDir);
@@ -8,6 +8,7 @@ import {
8
8
 
9
9
  const TIMINGS_FILENAME = "timings.json";
10
10
  const RESULT_ARTIFACTS_DIRNAME = "artifacts";
11
+ const RESULT_LOGS_DIRNAME = "logs";
11
12
 
12
13
  export function writeRunArtifact(productDir, artifact) {
13
14
  const resultsDir = path.join(productDir, ".testkit", "results");
@@ -27,6 +28,10 @@ export function resetResultArtifacts(productDir) {
27
28
  recursive: true,
28
29
  force: true,
29
30
  });
31
+ fs.rmSync(path.join(productDir, ".testkit", "results", RESULT_LOGS_DIRNAME), {
32
+ recursive: true,
33
+ force: true,
34
+ });
30
35
  }
31
36
 
32
37
  export function persistTaskArtifacts(productDir, task, emittedArtifacts) {
@@ -81,6 +86,27 @@ export function persistTaskArtifacts(productDir, task, emittedArtifacts) {
81
86
  });
82
87
  }
83
88
 
89
+ export function persistTaskOutputArtifacts(productDir, task, outputs) {
90
+ if (!Array.isArray(outputs) || outputs.length === 0) return [];
91
+ return persistTaskArtifacts(
92
+ productDir,
93
+ task,
94
+ outputs
95
+ .filter((entry) => typeof entry?.text === "string" && entry.text.trim().length > 0)
96
+ .map((entry) => ({
97
+ name: entry.name || "task-output",
98
+ kind: entry.kind || "runtime.output",
99
+ summary: entry.summary || summarizeOutput(entry.text),
100
+ contentType: entry.contentType || "text/plain",
101
+ emittedAt: entry.emittedAt || new Date().toISOString(),
102
+ data: {
103
+ stream: entry.stream || null,
104
+ text: entry.text,
105
+ },
106
+ }))
107
+ );
108
+ }
109
+
84
110
  export function loadTimings(productDir) {
85
111
  const filePath = path.join(productDir, ".testkit", TIMINGS_FILENAME);
86
112
  if (!fs.existsSync(filePath)) {
@@ -113,3 +139,12 @@ function sanitizePathSegment(value) {
113
139
  function normalizePath(filePath) {
114
140
  return filePath.split(path.sep).join("/");
115
141
  }
142
+
143
+ function summarizeOutput(text) {
144
+ const firstLine = String(text)
145
+ .split(/\r?\n/)
146
+ .map((line) => line.trim())
147
+ .find(Boolean);
148
+ if (!firstLine) return "captured task output";
149
+ return firstLine.slice(0, 120);
150
+ }
@@ -1,3 +1,6 @@
1
+ import path from "path";
2
+ import { fileURLToPath } from "url";
3
+
1
4
  export function determineDefaultRuntimeFailure(result, summary, firstLine) {
2
5
  const fatalRuntimeError = extractDefaultRuntimeFatalError(result.stderr || "", firstLine);
3
6
  if (fatalRuntimeError) {
@@ -26,6 +29,22 @@ export function extractDefaultRuntimeFatalError(stderr, firstLine) {
26
29
  return matched?.[1]?.trim() || firstLine(stderr);
27
30
  }
28
31
 
32
+ export function extractDefaultRuntimeFatalDetail(stderr, firstLine) {
33
+ if (!stderr || !/source=stacktrace/.test(stderr)) return null;
34
+ const message = extractDefaultRuntimeFatalError(stderr, firstLine);
35
+ if (!message) return null;
36
+
37
+ const location = extractFirstLocation(stderr);
38
+ return {
39
+ kind: "runtime-exception",
40
+ key: location ? `${location.path}:${location.line}:${location.column}` : message,
41
+ title: "Uncaught runtime exception",
42
+ message: `Uncaught testkit suite error: ${message}`,
43
+ location,
44
+ stack: sanitizeStack(stderr),
45
+ };
46
+ }
47
+
29
48
  export function extractDefaultRuntimeThresholdFailures(summary) {
30
49
  const metrics = summary?.metrics;
31
50
  if (!metrics || typeof metrics !== "object") return [];
@@ -51,3 +70,50 @@ export function sanitizeDefaultRuntimeExitError(exitCode, output, firstLine) {
51
70
  }
52
71
  return `Default runtime failed with exit code ${exitCode}`;
53
72
  }
73
+
74
+ function extractFirstLocation(stderr) {
75
+ const locations = [...String(stderr).matchAll(/(file:\/\/[^\s)]+|\/[^\s):]+):(\d+):(\d+)/g)].map((match) => ({
76
+ path: normalizeLocationPath(match[1]),
77
+ line: Number(match[2]),
78
+ column: Number(match[3]),
79
+ }));
80
+ if (locations.length === 0) return null;
81
+
82
+ return (
83
+ locations.find((location) => isPreferredLocation(location.path)) ||
84
+ locations.find((location) => !isRuntimeInternalLocation(location.path)) ||
85
+ locations[0]
86
+ );
87
+ }
88
+
89
+ function normalizeLocationPath(rawPath) {
90
+ if (!rawPath) return rawPath;
91
+ if (rawPath.startsWith("file://")) {
92
+ try {
93
+ return fileURLToPath(rawPath);
94
+ } catch {
95
+ return rawPath;
96
+ }
97
+ }
98
+ return rawPath;
99
+ }
100
+
101
+ function sanitizeStack(stderr) {
102
+ return String(stderr)
103
+ .split(/\r?\n/)
104
+ .map((line) => line.trim())
105
+ .filter(Boolean)
106
+ .join("\n");
107
+ }
108
+
109
+ function isPreferredLocation(filePath) {
110
+ if (typeof filePath !== "string") return false;
111
+ const normalized = filePath.split(path.sep).join("/");
112
+ return normalized.includes("/.testkit/_bundles/") || normalized.includes("/__testkit__/");
113
+ }
114
+
115
+ function isRuntimeInternalLocation(filePath) {
116
+ if (typeof filePath !== "string") return false;
117
+ const normalized = filePath.split(path.sep).join("/");
118
+ return normalized.includes("/testkit/lib/runtime-src/k6/") || normalized.includes("/vendor/k6");
119
+ }
@@ -7,15 +7,18 @@ import {
7
7
  buildFileTimeoutEnv,
8
8
  formatFileTimeoutBudgetError,
9
9
  } from "../shared/file-timeout.mjs";
10
- import { persistTaskArtifacts } from "./artifacts.mjs";
11
- import { determineDefaultRuntimeFailure } from "./default-runtime-errors.mjs";
10
+ import { persistTaskArtifacts, persistTaskOutputArtifacts } from "./artifacts.mjs";
11
+ import {
12
+ determineDefaultRuntimeFailure,
13
+ extractDefaultRuntimeFatalDetail,
14
+ } from "./default-runtime-errors.mjs";
12
15
  import { collectFailureDetailsFromRuntimeArtifacts } from "./failure-details.mjs";
13
16
  import { RUNTIME_ARTIFACT_MARKER } from "../runtime-src/k6/artifacts.js";
14
17
  import { readDatabaseUrl } from "./state-io.mjs";
15
18
  import { buildTaskExecutionEnv } from "./template.mjs";
16
19
  import { killChildProcess } from "./processes.mjs";
17
20
 
18
- export async function runHttpK6Task(targetConfig, task, lifecycle, lease) {
21
+ export async function runHttpK6Task(targetConfig, task, lifecycle, lease, reporter = null) {
19
22
  const baseUrl = targetConfig.testkit.local?.baseUrl;
20
23
  if (!baseUrl) {
21
24
  throw new Error(`Service "${targetConfig.name}" requires local.baseUrl for HTTP suites`);
@@ -34,11 +37,12 @@ export async function runHttpK6Task(targetConfig, task, lifecycle, lease) {
34
37
  task,
35
38
  lease,
36
39
  ["run", "--address", "127.0.0.1:0", "-e", `BASE_URL=${baseUrl}`, bundledFile],
37
- lifecycle
40
+ lifecycle,
41
+ reporter
38
42
  );
39
43
  }
40
44
 
41
- export async function runDalTask(targetConfig, task, lifecycle, lease) {
45
+ export async function runDalTask(targetConfig, task, lifecycle, lease, reporter = null) {
42
46
  const databaseUrl = readDatabaseUrl(targetConfig.stateDir);
43
47
  if (!databaseUrl) {
44
48
  throw new Error(`Service "${targetConfig.name}" requires a database for DAL suites`);
@@ -57,12 +61,27 @@ export async function runDalTask(targetConfig, task, lifecycle, lease) {
57
61
  task,
58
62
  lease,
59
63
  ["run", "--address", "127.0.0.1:0", "-e", `DATABASE_URL=${databaseUrl}`, bundledFile],
60
- lifecycle
64
+ lifecycle,
65
+ reporter
61
66
  );
62
67
  }
63
68
 
64
- export async function runDefaultRuntimeTask(targetConfig, task, lease, args, lifecycle, firstLine) {
69
+ export async function runDefaultRuntimeTask(
70
+ targetConfig,
71
+ task,
72
+ lease,
73
+ args,
74
+ lifecycle,
75
+ reporterOrFirstLine,
76
+ maybeFirstLine
77
+ ) {
65
78
  const k6Binary = resolveK6Binary();
79
+ const reporter =
80
+ reporterOrFirstLine && typeof reporterOrFirstLine === "object"
81
+ ? reporterOrFirstLine
82
+ : null;
83
+ const firstLine =
84
+ typeof reporterOrFirstLine === "function" ? reporterOrFirstLine : maybeFirstLine;
66
85
  const getFirstLine = firstLine || defaultFirstLine;
67
86
  const summaryFile = buildDefaultRuntimeSummaryPath(lease, task);
68
87
  fs.mkdirSync(path.dirname(summaryFile), { recursive: true });
@@ -91,7 +110,7 @@ export async function runDefaultRuntimeTask(targetConfig, task, lease, args, lif
91
110
  if (subprocess.pid) interruptSubprocess();
92
111
  else subprocess.once?.("spawn", interruptSubprocess);
93
112
  }
94
- console.log(`·· ${targetConfig.runtimeLabel}:${task.suiteName} → ${task.file}`);
113
+ reporter?.taskStarted?.(task, targetConfig);
95
114
  let result;
96
115
  let timedOut;
97
116
  try {
@@ -102,8 +121,6 @@ export async function runDefaultRuntimeTask(targetConfig, task, lease, args, lif
102
121
 
103
122
  const stdout = parseDefaultRuntimeOutput(result.stdout || "");
104
123
  const stderr = parseDefaultRuntimeOutput(result.stderr || "");
105
- if (stdout.visibleOutput) process.stdout.write(stdout.visibleOutput);
106
- if (stderr.visibleOutput) process.stderr.write(stderr.visibleOutput);
107
124
 
108
125
  const summary = readDefaultRuntimeSummary(summaryFile);
109
126
  const rawRuntimeArtifacts = [...stdout.artifacts, ...stderr.artifacts];
@@ -112,7 +129,31 @@ export async function runDefaultRuntimeTask(targetConfig, task, lease, args, lif
112
129
  task,
113
130
  rawRuntimeArtifacts
114
131
  );
132
+ const outputArtifacts = persistTaskOutputArtifacts(targetConfig.productDir, task, [
133
+ stdout.visibleOutput
134
+ ? {
135
+ name: "default-runtime-stdout",
136
+ kind: "runtime.output",
137
+ summary: defaultFirstLine(stdout.visibleOutput) || "captured stdout",
138
+ stream: "stdout",
139
+ text: stdout.visibleOutput,
140
+ }
141
+ : null,
142
+ stderr.visibleOutput
143
+ ? {
144
+ name: "default-runtime-stderr",
145
+ kind: "runtime.output",
146
+ summary: defaultFirstLine(stderr.visibleOutput) || "captured stderr",
147
+ stream: "stderr",
148
+ text: stderr.visibleOutput,
149
+ }
150
+ : null,
151
+ ]);
115
152
  const failureDetails = collectFailureDetailsFromRuntimeArtifacts(rawRuntimeArtifacts);
153
+ const fatalRuntimeDetail = extractDefaultRuntimeFatalDetail(result.stderr || "", getFirstLine);
154
+ if (fatalRuntimeDetail && !failureDetails.some((detail) => detail?.kind === "runtime-exception")) {
155
+ failureDetails.unshift(fatalRuntimeDetail);
156
+ }
116
157
  const runtimeError = timedOut
117
158
  ? formatFileTimeoutBudgetError(fileTimeoutSeconds)
118
159
  : determineDefaultRuntimeFailure(result, summary, getFirstLine);
@@ -125,7 +166,7 @@ export async function runDefaultRuntimeTask(targetConfig, task, lease, args, lif
125
166
  durationMs: finishedAt - startedAt,
126
167
  startedAt,
127
168
  finishedAt,
128
- artifacts: runtimeArtifacts,
169
+ artifacts: [...runtimeArtifacts, ...outputArtifacts],
129
170
  failureDetails,
130
171
  };
131
172
  }
@@ -28,6 +28,24 @@ export function normalizeFailureDetail(detail) {
28
28
  const message = normalizeNonEmptyString(detail.message);
29
29
  if (message) normalized.message = message;
30
30
 
31
+ if (detail.expected !== undefined) normalized.expected = cloneJsonValue(detail.expected);
32
+ if (detail.actual !== undefined) normalized.actual = cloneJsonValue(detail.actual);
33
+
34
+ const traceId = normalizeNonEmptyString(detail.traceId);
35
+ if (traceId) normalized.traceId = traceId;
36
+
37
+ const request = normalizeRecord(detail.request);
38
+ if (request) normalized.request = request;
39
+
40
+ const response = normalizeRecord(detail.response);
41
+ if (response) normalized.response = response;
42
+
43
+ const location = normalizeRecord(detail.location);
44
+ if (location) normalized.location = location;
45
+
46
+ const stack = normalizeNonEmptyString(detail.stack);
47
+ if (stack) normalized.stack = stack;
48
+
31
49
  return normalized;
32
50
  }
33
51
 
@@ -89,3 +107,16 @@ function normalizePositiveInteger(value) {
89
107
  if (!Number.isInteger(value) || value <= 0) return null;
90
108
  return value;
91
109
  }
110
+
111
+ function normalizeRecord(value) {
112
+ if (!value || typeof value !== "object" || Array.isArray(value)) return null;
113
+ return cloneJsonValue(value);
114
+ }
115
+
116
+ function cloneJsonValue(value) {
117
+ try {
118
+ return JSON.parse(JSON.stringify(value));
119
+ } catch {
120
+ return String(value);
121
+ }
122
+ }
@@ -60,4 +60,55 @@ describe("runner failure details", () => {
60
60
  },
61
61
  ]);
62
62
  });
63
+
64
+ it("preserves rich assertion metadata", () => {
65
+ expect(
66
+ mergeFailureDetails([
67
+ {
68
+ kind: "http-assertion",
69
+ key: "GET /health > status is 200",
70
+ title: "status is 200",
71
+ expected: 200,
72
+ actual: 404,
73
+ request: {
74
+ method: "GET",
75
+ path: "/health",
76
+ requestId: "req-1",
77
+ },
78
+ response: {
79
+ status: 404,
80
+ bodyPreview: '{"error":"nope"}',
81
+ },
82
+ location: {
83
+ path: "/tmp/example.ts",
84
+ line: 12,
85
+ column: 4,
86
+ },
87
+ },
88
+ ])
89
+ ).toEqual([
90
+ {
91
+ kind: "http-assertion",
92
+ key: "GET /health > status is 200",
93
+ title: "status is 200",
94
+ count: 1,
95
+ expected: 200,
96
+ actual: 404,
97
+ request: {
98
+ method: "GET",
99
+ path: "/health",
100
+ requestId: "req-1",
101
+ },
102
+ response: {
103
+ status: 404,
104
+ bodyPreview: '{"error":"nope"}',
105
+ },
106
+ location: {
107
+ path: "/tmp/example.ts",
108
+ line: 12,
109
+ column: 4,
110
+ },
111
+ },
112
+ ]);
113
+ });
63
114
  });