@elench/testkit 0.1.78 → 0.1.80
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/cli/presentation/colors.mjs +22 -5
- package/lib/cli/presentation/discovery-reporter.mjs +33 -12
- package/lib/cli/presentation/failure-presentation.mjs +31 -0
- package/lib/cli/presentation/run-reporter.mjs +65 -29
- package/lib/cli/presentation/run-reporter.test.mjs +132 -26
- package/lib/cli/presentation/summary-box.mjs +45 -0
- package/lib/cli/presentation/summary-box.test.mjs +43 -0
- package/lib/cli/presentation/terminal-layout.mjs +43 -0
- package/lib/cli/presentation/terminal-layout.test.mjs +23 -0
- package/lib/known-failures/github.mjs +6 -2
- package/lib/runner/formatting.mjs +77 -90
- package/lib/runner/formatting.test.mjs +90 -128
- package/lib/runner/orchestrator.mjs +2 -0
- package/lib/runner/worker-loop.mjs +1 -0
- package/node_modules/@elench/next-analysis/package.json +1 -1
- package/node_modules/@elench/testkit-bridge/package.json +2 -2
- package/node_modules/@elench/testkit-protocol/package.json +1 -1
- package/node_modules/@elench/ts-analysis/package.json +1 -1
- package/package.json +10 -6
|
@@ -1,12 +1,21 @@
|
|
|
1
1
|
import { createColors } from "picocolors";
|
|
2
|
+
import figures from "figures";
|
|
2
3
|
|
|
3
4
|
const pc = createColors(Boolean(process.stdout?.isTTY || process.env.FORCE_COLOR));
|
|
4
5
|
|
|
5
6
|
export function colorStatus(status) {
|
|
6
|
-
if (status === "PASS") return pc.green(
|
|
7
|
-
if (status === "FAIL") return pc.red(
|
|
8
|
-
if (status === "SKIP") return pc.yellow(
|
|
9
|
-
if (status === "RUN") return pc.cyan(
|
|
7
|
+
if (status === "PASS") return pc.green(figures.tick);
|
|
8
|
+
if (status === "FAIL") return pc.red(figures.cross);
|
|
9
|
+
if (status === "SKIP") return pc.yellow(figures.arrowDown);
|
|
10
|
+
if (status === "RUN") return pc.cyan(figures.play);
|
|
11
|
+
return status;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export function statusLabel(status) {
|
|
15
|
+
if (status === "PASS") return pc.green(`${figures.tick} PASS`);
|
|
16
|
+
if (status === "FAIL") return pc.red(`${figures.cross} FAIL`);
|
|
17
|
+
if (status === "SKIP") return pc.yellow(`${figures.arrowDown} SKIP`);
|
|
18
|
+
if (status === "RUN") return pc.cyan(`${figures.play} RUN`);
|
|
10
19
|
return status;
|
|
11
20
|
}
|
|
12
21
|
|
|
@@ -18,6 +27,14 @@ export function bold(text) {
|
|
|
18
27
|
return pc.bold(text);
|
|
19
28
|
}
|
|
20
29
|
|
|
30
|
+
export function boldRed(text) {
|
|
31
|
+
return pc.bold(pc.red(text));
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export function red(text) {
|
|
35
|
+
return pc.red(text);
|
|
36
|
+
}
|
|
37
|
+
|
|
21
38
|
export function muted(text) {
|
|
22
39
|
return pc.dim(text);
|
|
23
40
|
}
|
|
@@ -53,7 +70,7 @@ export function colorResultLine(line) {
|
|
|
53
70
|
}
|
|
54
71
|
|
|
55
72
|
export function colorSectionLine(line) {
|
|
56
|
-
if (line === "
|
|
73
|
+
if (line === "Known-failure issues:" || line === "Triage:") {
|
|
57
74
|
return pc.bold(line);
|
|
58
75
|
}
|
|
59
76
|
if (/^Summary: /.test(line)) return line.replace("Summary:", `${pc.bold("Summary:")}`);
|
|
@@ -5,13 +5,18 @@ import {
|
|
|
5
5
|
colorDiagnosticSeverity,
|
|
6
6
|
colorHeading,
|
|
7
7
|
colorService,
|
|
8
|
-
colorStatus,
|
|
9
8
|
colorTypeBadge,
|
|
10
9
|
muted,
|
|
10
|
+
statusLabel,
|
|
11
11
|
} from "./colors.mjs";
|
|
12
12
|
|
|
13
13
|
const TYPE_ORDER = ["int", "e2e", "scenario", "dal", "load", "pw"];
|
|
14
14
|
|
|
15
|
+
const TREE_BRANCH = "\u251C\u2500\u2500 ";
|
|
16
|
+
const TREE_LAST = "\u2514\u2500\u2500 ";
|
|
17
|
+
const TREE_PIPE = "\u2502 ";
|
|
18
|
+
const TREE_SPACE = " ";
|
|
19
|
+
|
|
15
20
|
export function buildDiscoveryReportLines(result, options = {}) {
|
|
16
21
|
const mode = options.outputMode || "compact";
|
|
17
22
|
return mode === "verbose" ? buildVerboseLines(result) : buildCompactLines(result);
|
|
@@ -20,7 +25,7 @@ export function buildDiscoveryReportLines(result, options = {}) {
|
|
|
20
25
|
function buildCompactLines(result) {
|
|
21
26
|
const lines = [];
|
|
22
27
|
lines.push(
|
|
23
|
-
`${colorHeading("Summary")} ${result.summary.files} files
|
|
28
|
+
`${colorHeading("Summary")} ${result.summary.files} files \u00B7 ${result.summary.activeFiles} active \u00B7 ${result.summary.skippedFiles} skipped \u00B7 ${result.summary.suites} suites \u00B7 ${result.summary.services} services`
|
|
24
29
|
);
|
|
25
30
|
|
|
26
31
|
if (result.history?.available) {
|
|
@@ -35,17 +40,33 @@ function buildCompactLines(result) {
|
|
|
35
40
|
for (const service of result.services) {
|
|
36
41
|
lines.push("");
|
|
37
42
|
lines.push(
|
|
38
|
-
`${colorService(service.name)} ${muted(`(${service.fileCount} files${service.dependsOn.length > 0 ? `
|
|
43
|
+
`${colorService(service.name)} ${muted(`(${service.fileCount} files${service.dependsOn.length > 0 ? ` \u00B7 depends on ${service.dependsOn.join(", ")}` : ""})`)}`
|
|
39
44
|
);
|
|
40
45
|
const typeGroups = suitesByService.get(service.name) || new Map();
|
|
41
|
-
|
|
46
|
+
const types = orderedTypes([...typeGroups.keys()]);
|
|
47
|
+
|
|
48
|
+
for (let ti = 0; ti < types.length; ti++) {
|
|
49
|
+
const type = types[ti];
|
|
50
|
+
const isLastType = ti === types.length - 1;
|
|
51
|
+
const typeConnector = isLastType ? TREE_LAST : TREE_BRANCH;
|
|
52
|
+
const typePrefix = isLastType ? TREE_SPACE : TREE_PIPE;
|
|
42
53
|
const suites = typeGroups.get(type) || [];
|
|
43
|
-
lines.push(
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
54
|
+
lines.push(`${typeConnector}${colorTypeBadge(type.toUpperCase())} ${formatSelectionTypeLabel(type)}`);
|
|
55
|
+
|
|
56
|
+
for (let si = 0; si < suites.length; si++) {
|
|
57
|
+
const suite = suites[si];
|
|
58
|
+
const isLastSuite = si === suites.length - 1;
|
|
59
|
+
const suiteConnector = isLastSuite ? TREE_LAST : TREE_BRANCH;
|
|
60
|
+
const suitePrefix = isLastSuite ? TREE_SPACE : TREE_PIPE;
|
|
61
|
+
lines.push(`${typePrefix}${suiteConnector}${bold(suite.groupLabel)} ${muted(`(${suite.fileCount} files)`)}`);
|
|
62
|
+
|
|
63
|
+
const files = filesBySuite.get(suite.id) || [];
|
|
64
|
+
for (let fi = 0; fi < files.length; fi++) {
|
|
65
|
+
const file = files[fi];
|
|
66
|
+
const isLastFile = fi === files.length - 1;
|
|
67
|
+
const fileConnector = isLastFile ? TREE_LAST : TREE_BRANCH;
|
|
68
|
+
const status = file.skipped ? `${statusLabel("SKIP")} ${file.skipReason}` : muted(buildHistoryHint(file));
|
|
69
|
+
lines.push(`${typePrefix}${suitePrefix}${fileConnector}${file.displayName}${status ? ` ${status}` : ""}`);
|
|
49
70
|
}
|
|
50
71
|
}
|
|
51
72
|
}
|
|
@@ -73,7 +94,7 @@ function buildVerboseLines(result) {
|
|
|
73
94
|
if (service.dependsOn.length > 0) {
|
|
74
95
|
lines.push(` dependsOn ${service.dependsOn.join(", ")}`);
|
|
75
96
|
}
|
|
76
|
-
lines.push(` files ${service.fileCount}
|
|
97
|
+
lines.push(` files ${service.fileCount} \u00B7 active ${service.activeFileCount} \u00B7 skipped ${service.skippedFileCount}`);
|
|
77
98
|
|
|
78
99
|
const suites = result.suites.filter((suite) => suite.service === service.name).sort(compareSuites);
|
|
79
100
|
for (const suite of suites) {
|
|
@@ -162,5 +183,5 @@ function buildHistoryHint(file) {
|
|
|
162
183
|
file.history.firstSeenAt ? `seen ${file.history.firstSeenAt.slice(0, 10)}` : null,
|
|
163
184
|
]
|
|
164
185
|
.filter(Boolean)
|
|
165
|
-
.join("
|
|
186
|
+
.join(" \u00B7 ");
|
|
166
187
|
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { buildFailurePresentation } from "../../runner/formatting.mjs";
|
|
2
|
+
import { renderIndentedBlock } from "./terminal-layout.mjs";
|
|
3
|
+
|
|
4
|
+
export function renderFailureBlock(task, outcome, { width, knownFailures } = {}) {
|
|
5
|
+
const presentation = buildFailurePresentation(
|
|
6
|
+
{
|
|
7
|
+
service: task.serviceName,
|
|
8
|
+
type: normalizeKnownFailureType(task),
|
|
9
|
+
path: task.file,
|
|
10
|
+
error: outcome.error || null,
|
|
11
|
+
failureDetails: Array.isArray(outcome.failureDetails) ? outcome.failureDetails : [],
|
|
12
|
+
suiteError: null,
|
|
13
|
+
},
|
|
14
|
+
knownFailures
|
|
15
|
+
);
|
|
16
|
+
|
|
17
|
+
const lines = [];
|
|
18
|
+
if (presentation.primary) {
|
|
19
|
+
lines.push(...renderIndentedBlock(presentation.primary, { width, indent: " " }));
|
|
20
|
+
}
|
|
21
|
+
for (const detail of presentation.details) {
|
|
22
|
+
lines.push(...renderIndentedBlock(detail, { width, indent: " " }));
|
|
23
|
+
}
|
|
24
|
+
return lines;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
function normalizeKnownFailureType(task) {
|
|
28
|
+
if (task.framework === "playwright") return "pw";
|
|
29
|
+
if (task.type === "integration") return "int";
|
|
30
|
+
return task.type;
|
|
31
|
+
}
|
|
@@ -1,15 +1,28 @@
|
|
|
1
1
|
import path from "path";
|
|
2
2
|
import {
|
|
3
|
-
|
|
3
|
+
buildRunSummaryData,
|
|
4
4
|
buildDebugRunSummaryLines,
|
|
5
5
|
formatDuration,
|
|
6
6
|
} from "../../runner/formatting.mjs";
|
|
7
|
-
import {
|
|
7
|
+
import { boldRed, colorSectionLine, dim, statusLabel } from "./colors.mjs";
|
|
8
|
+
import { renderFailureBlock } from "./failure-presentation.mjs";
|
|
9
|
+
import { renderSummaryBox } from "./summary-box.mjs";
|
|
10
|
+
import { getTerminalWidth } from "./terminal-layout.mjs";
|
|
8
11
|
|
|
9
12
|
export function createRunReporter({ outputMode = "compact", stdout = process.stdout, stderr = process.stderr } = {}) {
|
|
10
13
|
const mode = outputMode || "compact";
|
|
14
|
+
let completedCount = 0;
|
|
15
|
+
let totalFileCount = 0;
|
|
16
|
+
let knownFailures = null;
|
|
17
|
+
|
|
11
18
|
return {
|
|
12
19
|
outputMode: mode,
|
|
20
|
+
setTotalFileCount(count) {
|
|
21
|
+
totalFileCount = count;
|
|
22
|
+
},
|
|
23
|
+
setKnownFailures(document) {
|
|
24
|
+
knownFailures = document;
|
|
25
|
+
},
|
|
13
26
|
writeLine(line = "") {
|
|
14
27
|
stdout.write(`${line}\n`);
|
|
15
28
|
},
|
|
@@ -34,7 +47,7 @@ export function createRunReporter({ outputMode = "compact", stdout = process.std
|
|
|
34
47
|
if (operation.status === "failed") {
|
|
35
48
|
const detail = shortenMessage(operation.error || operation.summary || operation.stage);
|
|
36
49
|
stdout.write(
|
|
37
|
-
`${
|
|
50
|
+
`${statusLabel("FAIL")} ${"SETUP"} ${operation.serviceName} ${operation.stage} ${dim(duration)} ${boldRed(detail)}\n`
|
|
38
51
|
);
|
|
39
52
|
return;
|
|
40
53
|
}
|
|
@@ -44,7 +57,7 @@ export function createRunReporter({ outputMode = "compact", stdout = process.std
|
|
|
44
57
|
(operation.durationMs || 0) >= 5_000
|
|
45
58
|
) {
|
|
46
59
|
const summary = shortenMessage(operation.summary || operation.stage);
|
|
47
|
-
stdout.write(`${
|
|
60
|
+
stdout.write(`${statusLabel("RUN")} ${"SETUP"} ${operation.serviceName} ${summary} ${dim(duration)}\n`);
|
|
48
61
|
}
|
|
49
62
|
},
|
|
50
63
|
localServiceStarting(config, command) {
|
|
@@ -52,44 +65,76 @@ export function createRunReporter({ outputMode = "compact", stdout = process.std
|
|
|
52
65
|
stdout.write(`Starting ${config.runtimeLabel}:${config.name}: ${command}\n`);
|
|
53
66
|
},
|
|
54
67
|
serviceSkipped(config, reason) {
|
|
55
|
-
stdout.write(`${
|
|
68
|
+
stdout.write(`${statusLabel("SKIP")} ${config.name} ${reason}\n`);
|
|
56
69
|
},
|
|
57
70
|
plannedSkip(entry) {
|
|
58
71
|
stdout.write(
|
|
59
|
-
`${
|
|
72
|
+
`${statusLabel("SKIP")} ${entry.serviceName} ${entry.type} ${normalizePath(entry.file)} ${dim("0s")} ${shortenMessage(entry.reason || "skipped")}\n`
|
|
60
73
|
);
|
|
61
74
|
},
|
|
62
75
|
taskStarted(task, targetConfig) {
|
|
63
76
|
if (mode !== "debug") return;
|
|
64
|
-
stdout.write(`${
|
|
77
|
+
stdout.write(`${statusLabel("RUN")} ${targetConfig.name} ${task.type} ${task.file}\n`);
|
|
65
78
|
},
|
|
66
79
|
taskFinished(task, outcome) {
|
|
67
80
|
if (mode === "json") return;
|
|
81
|
+
completedCount += 1;
|
|
68
82
|
const status = outcome.status === "skipped" ? "SKIP" : outcome.failed ? "FAIL" : "PASS";
|
|
69
83
|
const duration = formatDuration(outcome.durationMs || 0);
|
|
70
|
-
const primaryFailure = firstFailureDetail(outcome);
|
|
71
|
-
const preferredFailure =
|
|
72
|
-
primaryFailure && isThresholdWrapperMessage(outcome.error) ? primaryFailure : outcome.error || primaryFailure;
|
|
73
84
|
const detail =
|
|
74
|
-
status === "
|
|
75
|
-
|
|
76
|
-
: outcome.status === "not_run"
|
|
77
|
-
? ` ${shortenMessage(outcome.reason || "not run")}`
|
|
85
|
+
outcome.status === "not_run"
|
|
86
|
+
? ` ${dim(shortenMessage(outcome.reason || "not run"))}`
|
|
78
87
|
: "";
|
|
88
|
+
const progress = mode === "compact" && totalFileCount > 0 ? `${dim(`[${completedCount}/${totalFileCount}]`)} ` : "";
|
|
79
89
|
stdout.write(
|
|
80
|
-
`${
|
|
90
|
+
`${progress}${statusLabel(status)} ${task.serviceName} ${displayTaskType(task)} ${normalizePath(task.file)} ${dim(duration)}${detail}\n`
|
|
81
91
|
);
|
|
92
|
+
if (status === "FAIL") {
|
|
93
|
+
const detailLines = renderFailureBlock(task, outcome, {
|
|
94
|
+
width: getTerminalWidth(stdout, 100),
|
|
95
|
+
knownFailures,
|
|
96
|
+
});
|
|
97
|
+
for (const line of detailLines) {
|
|
98
|
+
stdout.write(`${line}\n`);
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
},
|
|
102
|
+
runtimeError(task, message) {
|
|
103
|
+
if (mode === "json") return;
|
|
104
|
+
stdout.write(`${statusLabel("FAIL")} ${task.serviceName} runtime error\n`);
|
|
105
|
+
stdout.write(` ${boldRed(shortenMessage(message || "runtime error"))}\n`);
|
|
82
106
|
},
|
|
83
107
|
telemetry(message) {
|
|
84
108
|
if (mode === "json") return;
|
|
85
109
|
stdout.write(`${message}\n`);
|
|
86
110
|
},
|
|
87
111
|
runSummary(results, durationMs, knownFailureIssueValidation = null) {
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
112
|
+
if (mode === "debug") {
|
|
113
|
+
const lines = buildDebugRunSummaryLines(results, durationMs, knownFailureIssueValidation);
|
|
114
|
+
stdout.write("\n");
|
|
115
|
+
for (const line of lines) stdout.write(`${colorSectionLine(line)}\n`);
|
|
116
|
+
return;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
const summary = buildRunSummaryData(results, durationMs, knownFailureIssueValidation);
|
|
120
|
+
const rows = [
|
|
121
|
+
["Result", summary.result],
|
|
122
|
+
["Passed", String(summary.passed)],
|
|
123
|
+
["Failed", String(summary.failed)],
|
|
124
|
+
["Skipped", String(summary.skipped)],
|
|
125
|
+
["Not run", String(summary.notRun)],
|
|
126
|
+
["Files", String(summary.files)],
|
|
127
|
+
["Duration", summary.duration],
|
|
128
|
+
];
|
|
129
|
+
if (summary.serviceErrors > 0) {
|
|
130
|
+
rows.push(["Runtime errors", String(summary.serviceErrors)]);
|
|
131
|
+
}
|
|
132
|
+
if (summary.knownFailureIssues.length > 0) {
|
|
133
|
+
rows.push(["Known issues", summary.knownFailureIssues.join(", ")]);
|
|
134
|
+
}
|
|
135
|
+
const boxed = renderSummaryBox(rows, { stdout });
|
|
136
|
+
stdout.write("\n");
|
|
137
|
+
for (const line of boxed) stdout.write(`${line}\n`);
|
|
93
138
|
},
|
|
94
139
|
error(message) {
|
|
95
140
|
stderr.write(`${message}\n`);
|
|
@@ -107,15 +152,6 @@ function shortenMessage(message) {
|
|
|
107
152
|
return String(message).replace(/\s+/g, " ").trim().slice(0, 180);
|
|
108
153
|
}
|
|
109
154
|
|
|
110
|
-
function firstFailureDetail(outcome) {
|
|
111
|
-
const detail = outcome.failureDetails?.[0];
|
|
112
|
-
return detail?.message || detail?.title || null;
|
|
113
|
-
}
|
|
114
|
-
|
|
115
|
-
function isThresholdWrapperMessage(message) {
|
|
116
|
-
return /Default runtime thresholds failed:/.test(String(message || ""));
|
|
117
|
-
}
|
|
118
|
-
|
|
119
155
|
function normalizePath(filePath) {
|
|
120
156
|
return String(filePath).split(path.sep).join("/");
|
|
121
157
|
}
|
|
@@ -1,18 +1,31 @@
|
|
|
1
1
|
import { Writable } from "stream";
|
|
2
|
+
import figures from "figures";
|
|
2
3
|
import { describe, expect, it } from "vitest";
|
|
3
4
|
import { createRunReporter } from "./run-reporter.mjs";
|
|
4
5
|
|
|
6
|
+
function createCapture(columns = 100) {
|
|
7
|
+
let stdout = "";
|
|
8
|
+
const stream = new Writable({
|
|
9
|
+
write(chunk, _encoding, callback) {
|
|
10
|
+
stdout += chunk.toString();
|
|
11
|
+
callback();
|
|
12
|
+
},
|
|
13
|
+
});
|
|
14
|
+
stream.columns = columns;
|
|
15
|
+
return {
|
|
16
|
+
stream,
|
|
17
|
+
read() {
|
|
18
|
+
return stdout;
|
|
19
|
+
},
|
|
20
|
+
};
|
|
21
|
+
}
|
|
22
|
+
|
|
5
23
|
describe("run reporter setup output", () => {
|
|
6
24
|
it("prints concise high-level setup summaries in compact mode", () => {
|
|
7
|
-
|
|
25
|
+
const capture = createCapture();
|
|
8
26
|
const reporter = createRunReporter({
|
|
9
27
|
outputMode: "compact",
|
|
10
|
-
stdout:
|
|
11
|
-
write(chunk, _encoding, callback) {
|
|
12
|
-
stdout += chunk.toString();
|
|
13
|
-
callback();
|
|
14
|
-
},
|
|
15
|
-
}),
|
|
28
|
+
stdout: capture.stream,
|
|
16
29
|
});
|
|
17
30
|
|
|
18
31
|
reporter.setupOperationFinished({
|
|
@@ -24,20 +37,15 @@ describe("run reporter setup output", () => {
|
|
|
24
37
|
durationMs: 8_000,
|
|
25
38
|
});
|
|
26
39
|
|
|
27
|
-
expect(
|
|
28
|
-
expect(
|
|
40
|
+
expect(capture.read()).toContain(`${figures.play} RUN SETUP api template rebuild`);
|
|
41
|
+
expect(capture.read()).toContain("8s");
|
|
29
42
|
});
|
|
30
43
|
|
|
31
44
|
it("does not print low-level setup steps in compact mode", () => {
|
|
32
|
-
|
|
45
|
+
const capture = createCapture();
|
|
33
46
|
const reporter = createRunReporter({
|
|
34
47
|
outputMode: "compact",
|
|
35
|
-
stdout:
|
|
36
|
-
write(chunk, _encoding, callback) {
|
|
37
|
-
stdout += chunk.toString();
|
|
38
|
-
callback();
|
|
39
|
-
},
|
|
40
|
-
}),
|
|
48
|
+
stdout: capture.stream,
|
|
41
49
|
});
|
|
42
50
|
|
|
43
51
|
reporter.setupOperationFinished({
|
|
@@ -49,19 +57,14 @@ describe("run reporter setup output", () => {
|
|
|
49
57
|
durationMs: 8_000,
|
|
50
58
|
});
|
|
51
59
|
|
|
52
|
-
expect(
|
|
60
|
+
expect(capture.read()).toBe("");
|
|
53
61
|
});
|
|
54
62
|
|
|
55
63
|
it("prints concise setup failures", () => {
|
|
56
|
-
|
|
64
|
+
const capture = createCapture();
|
|
57
65
|
const reporter = createRunReporter({
|
|
58
66
|
outputMode: "compact",
|
|
59
|
-
stdout:
|
|
60
|
-
write(chunk, _encoding, callback) {
|
|
61
|
-
stdout += chunk.toString();
|
|
62
|
-
callback();
|
|
63
|
-
},
|
|
64
|
-
}),
|
|
67
|
+
stdout: capture.stream,
|
|
65
68
|
});
|
|
66
69
|
|
|
67
70
|
reporter.setupOperationFinished({
|
|
@@ -74,7 +77,110 @@ describe("run reporter setup output", () => {
|
|
|
74
77
|
error: "Command failed with exit code 1: node scripts/fail-prepare.mjs",
|
|
75
78
|
});
|
|
76
79
|
|
|
77
|
-
expect(
|
|
78
|
-
expect(
|
|
80
|
+
expect(capture.read()).toContain(`${figures.cross} FAIL SETUP api runtime:prepare`);
|
|
81
|
+
expect(capture.read()).toContain("Command failed with exit code 1");
|
|
82
|
+
});
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
describe("run reporter task output", () => {
|
|
86
|
+
it("prints failure details on lines beneath the failed file result", () => {
|
|
87
|
+
const capture = createCapture(88);
|
|
88
|
+
const reporter = createRunReporter({
|
|
89
|
+
outputMode: "compact",
|
|
90
|
+
stdout: capture.stream,
|
|
91
|
+
});
|
|
92
|
+
|
|
93
|
+
reporter.setTotalFileCount(3);
|
|
94
|
+
reporter.taskFinished(
|
|
95
|
+
{
|
|
96
|
+
serviceName: "api",
|
|
97
|
+
type: "integration",
|
|
98
|
+
framework: "k6",
|
|
99
|
+
file: "__testkit__/health/health.int.testkit.ts",
|
|
100
|
+
},
|
|
101
|
+
{
|
|
102
|
+
status: "failed",
|
|
103
|
+
failed: true,
|
|
104
|
+
durationMs: 4_000,
|
|
105
|
+
error: "Default runtime thresholds failed: checks(rate==1.0)",
|
|
106
|
+
failureDetails: [
|
|
107
|
+
{
|
|
108
|
+
kind: "http-assertion",
|
|
109
|
+
title: "status is 200",
|
|
110
|
+
message: "GET /health expected 200, got 404",
|
|
111
|
+
request: {
|
|
112
|
+
method: "GET",
|
|
113
|
+
path: "/health",
|
|
114
|
+
requestId: "req-1",
|
|
115
|
+
},
|
|
116
|
+
response: {
|
|
117
|
+
status: 404,
|
|
118
|
+
bodyPreview: '{"error":"route not found"}',
|
|
119
|
+
},
|
|
120
|
+
},
|
|
121
|
+
],
|
|
122
|
+
}
|
|
123
|
+
);
|
|
124
|
+
|
|
125
|
+
const lines = capture.read().trimEnd().split("\n");
|
|
126
|
+
expect(lines[0]).toContain("[1/3]");
|
|
127
|
+
expect(lines[0]).toContain(`${figures.cross} FAIL api int __testkit__/health/health.int.testkit.ts 4s`);
|
|
128
|
+
expect(lines[1]).toBe(" GET /health expected 200, got 404");
|
|
129
|
+
expect(lines[2]).toContain(' response: {"error":"route not found"}');
|
|
130
|
+
expect(lines[3]).toBe(" triage: untriaged");
|
|
131
|
+
expect(lines[4]).toBe(" logs: requestId=req-1");
|
|
132
|
+
});
|
|
133
|
+
|
|
134
|
+
it("renders a compact aggregate summary box", () => {
|
|
135
|
+
const capture = createCapture(72);
|
|
136
|
+
const reporter = createRunReporter({
|
|
137
|
+
outputMode: "compact",
|
|
138
|
+
stdout: capture.stream,
|
|
139
|
+
});
|
|
140
|
+
|
|
141
|
+
reporter.runSummary(
|
|
142
|
+
[
|
|
143
|
+
{
|
|
144
|
+
name: "api",
|
|
145
|
+
skipped: false,
|
|
146
|
+
failed: true,
|
|
147
|
+
suiteCount: 1,
|
|
148
|
+
completedSuiteCount: 1,
|
|
149
|
+
skippedSuiteCount: 0,
|
|
150
|
+
failedSuiteCount: 1,
|
|
151
|
+
totalFileCount: 3,
|
|
152
|
+
passedFileCount: 2,
|
|
153
|
+
failedFileCount: 1,
|
|
154
|
+
skippedFileCount: 0,
|
|
155
|
+
notRunFileCount: 0,
|
|
156
|
+
durationMs: 10_000,
|
|
157
|
+
suites: [],
|
|
158
|
+
errors: [],
|
|
159
|
+
},
|
|
160
|
+
],
|
|
161
|
+
20_000,
|
|
162
|
+
{
|
|
163
|
+
summary: {
|
|
164
|
+
byCode: {
|
|
165
|
+
closed_but_failing: 2,
|
|
166
|
+
state_mismatch: 1,
|
|
167
|
+
},
|
|
168
|
+
},
|
|
169
|
+
}
|
|
170
|
+
);
|
|
171
|
+
|
|
172
|
+
const output = capture.read();
|
|
173
|
+
expect(output).toContain("┌");
|
|
174
|
+
expect(output).toContain("│ Result");
|
|
175
|
+
expect(output).toContain("FAILED");
|
|
176
|
+
expect(output).toContain("│ Passed");
|
|
177
|
+
expect(output).toContain("│ Failed");
|
|
178
|
+
expect(output).toContain("│ Skipped");
|
|
179
|
+
expect(output).toContain("│ Not run");
|
|
180
|
+
expect(output).toContain("│ Files");
|
|
181
|
+
expect(output).toContain("│ Duration");
|
|
182
|
+
expect(output).toContain("│ Known issues");
|
|
183
|
+
expect(output).not.toContain("Failures:");
|
|
184
|
+
expect(output).not.toContain("Runtime Errors:");
|
|
79
185
|
});
|
|
80
186
|
});
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import figures from "figures";
|
|
2
|
+
import { clamp, getTerminalWidth, measureWidth, padEndVisible, wrapText } from "./terminal-layout.mjs";
|
|
3
|
+
|
|
4
|
+
export function renderSummaryBox(
|
|
5
|
+
rows,
|
|
6
|
+
{
|
|
7
|
+
stdout = process.stdout,
|
|
8
|
+
widthRatio = 0.55,
|
|
9
|
+
minWidth = 30,
|
|
10
|
+
maxWidth = 56,
|
|
11
|
+
minValueWidth = 8,
|
|
12
|
+
maxKeyWidth = 12,
|
|
13
|
+
} = {}
|
|
14
|
+
) {
|
|
15
|
+
if (!Array.isArray(rows) || rows.length === 0) return [];
|
|
16
|
+
|
|
17
|
+
const terminalWidth = getTerminalWidth(stdout, 100);
|
|
18
|
+
const preferredWidth = clamp(Math.floor(terminalWidth * widthRatio), minWidth, maxWidth);
|
|
19
|
+
const maxRenderableWidth = Math.max(minWidth, Math.min(preferredWidth, terminalWidth - 1));
|
|
20
|
+
const keyWidth = Math.min(
|
|
21
|
+
maxKeyWidth,
|
|
22
|
+
Math.max(...rows.map(([label]) => measureWidth(label)), 6)
|
|
23
|
+
);
|
|
24
|
+
const minBoxWidth = keyWidth + minValueWidth + 7;
|
|
25
|
+
const boxWidth = Math.max(minBoxWidth, maxRenderableWidth);
|
|
26
|
+
const valueWidth = Math.max(minValueWidth, boxWidth - keyWidth - 7);
|
|
27
|
+
|
|
28
|
+
const top = `${figures.lineDownRight}${figures.line.repeat(boxWidth - 2)}${figures.lineDownLeft}`;
|
|
29
|
+
const bottom = `${figures.lineUpRight}${figures.line.repeat(boxWidth - 2)}${figures.lineUpLeft}`;
|
|
30
|
+
const rendered = [top];
|
|
31
|
+
|
|
32
|
+
for (const [label, value] of rows) {
|
|
33
|
+
const wrappedValueLines = wrapText(value, valueWidth);
|
|
34
|
+
for (let index = 0; index < wrappedValueLines.length; index += 1) {
|
|
35
|
+
const keyCell = index === 0 ? padEndVisible(label, keyWidth) : " ".repeat(keyWidth);
|
|
36
|
+
const valueCell = padEndVisible(wrappedValueLines[index], valueWidth);
|
|
37
|
+
rendered.push(
|
|
38
|
+
`${figures.lineVertical} ${keyCell} ${figures.lineVertical} ${valueCell} ${figures.lineVertical}`
|
|
39
|
+
);
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
rendered.push(bottom);
|
|
44
|
+
return rendered;
|
|
45
|
+
}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { Writable } from "stream";
|
|
2
|
+
import { describe, expect, it } from "vitest";
|
|
3
|
+
import { renderSummaryBox } from "./summary-box.mjs";
|
|
4
|
+
|
|
5
|
+
function createStream(columns) {
|
|
6
|
+
const stream = new Writable({
|
|
7
|
+
write(_chunk, _encoding, callback) {
|
|
8
|
+
callback();
|
|
9
|
+
},
|
|
10
|
+
});
|
|
11
|
+
stream.columns = columns;
|
|
12
|
+
return stream;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
describe("summary box", () => {
|
|
16
|
+
it("renders key/value rows inside a bounded box", () => {
|
|
17
|
+
const lines = renderSummaryBox(
|
|
18
|
+
[
|
|
19
|
+
["Result", "FAILED"],
|
|
20
|
+
["Passed", "203"],
|
|
21
|
+
["Duration", "8m 56s"],
|
|
22
|
+
],
|
|
23
|
+
{ stdout: createStream(80) }
|
|
24
|
+
);
|
|
25
|
+
|
|
26
|
+
expect(lines[0]).toMatch(/^┌/);
|
|
27
|
+
expect(lines.at(-1)).toMatch(/^└/);
|
|
28
|
+
expect(lines.join("\n")).toContain("Result");
|
|
29
|
+
expect(lines.join("\n")).toContain("FAILED");
|
|
30
|
+
});
|
|
31
|
+
|
|
32
|
+
it("wraps long values instead of widening to content length", () => {
|
|
33
|
+
const lines = renderSummaryBox(
|
|
34
|
+
[["Known issues", "2 closed issues still failing, 6 state mismatches, used stale GitHub cache"]],
|
|
35
|
+
{ stdout: createStream(40) }
|
|
36
|
+
);
|
|
37
|
+
|
|
38
|
+
expect(lines.length).toBeGreaterThan(4);
|
|
39
|
+
expect(lines.join("\n")).toContain("Known issues");
|
|
40
|
+
expect(lines.join("\n")).toContain("state");
|
|
41
|
+
expect(lines.join("\n")).toContain("mismatches");
|
|
42
|
+
});
|
|
43
|
+
});
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import stringWidth from "string-width";
|
|
2
|
+
import stripAnsi from "strip-ansi";
|
|
3
|
+
import wrapAnsi from "wrap-ansi";
|
|
4
|
+
|
|
5
|
+
export function getTerminalWidth(stream = process.stdout, fallback = 100) {
|
|
6
|
+
const columns = Number(stream?.columns);
|
|
7
|
+
if (Number.isFinite(columns) && columns > 0) return columns;
|
|
8
|
+
return fallback;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export function measureWidth(text) {
|
|
12
|
+
return stringWidth(String(text ?? ""));
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export function stripTerminalFormatting(text) {
|
|
16
|
+
return stripAnsi(String(text ?? ""));
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export function padEndVisible(text, width) {
|
|
20
|
+
const normalized = String(text ?? "");
|
|
21
|
+
const padding = Math.max(0, width - measureWidth(normalized));
|
|
22
|
+
return `${normalized}${" ".repeat(padding)}`;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export function wrapText(text, width) {
|
|
26
|
+
const normalized = String(text ?? "");
|
|
27
|
+
if (width <= 0) return [normalized];
|
|
28
|
+
return wrapAnsi(normalized, width, {
|
|
29
|
+
hard: true,
|
|
30
|
+
trim: false,
|
|
31
|
+
wordWrap: true,
|
|
32
|
+
}).split("\n");
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export function renderIndentedBlock(text, { width, indent = " " } = {}) {
|
|
36
|
+
const visibleIndent = measureWidth(indent);
|
|
37
|
+
const contentWidth = Math.max(12, width - visibleIndent);
|
|
38
|
+
return wrapText(text, contentWidth).map((line) => `${indent}${line}`);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export function clamp(value, min, max) {
|
|
42
|
+
return Math.max(min, Math.min(max, value));
|
|
43
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { describe, expect, it } from "vitest";
|
|
2
|
+
import { getTerminalWidth, renderIndentedBlock, wrapText } from "./terminal-layout.mjs";
|
|
3
|
+
|
|
4
|
+
describe("terminal layout", () => {
|
|
5
|
+
it("falls back to a default width when stream columns are unavailable", () => {
|
|
6
|
+
expect(getTerminalWidth({}, 91)).toBe(91);
|
|
7
|
+
});
|
|
8
|
+
|
|
9
|
+
it("wraps indented blocks to the provided width", () => {
|
|
10
|
+
const lines = renderIndentedBlock("response: abcdefghijklmnopqrstuvwxyz", {
|
|
11
|
+
width: 20,
|
|
12
|
+
indent: " ",
|
|
13
|
+
});
|
|
14
|
+
|
|
15
|
+
expect(lines[0].startsWith(" ")).toBe(true);
|
|
16
|
+
expect(lines.length).toBeGreaterThan(1);
|
|
17
|
+
});
|
|
18
|
+
|
|
19
|
+
it("hard-wraps long unbroken values", () => {
|
|
20
|
+
const lines = wrapText("abcdefghijklmnopqrstuvwxyz", 8);
|
|
21
|
+
expect(lines.length).toBeGreaterThan(1);
|
|
22
|
+
});
|
|
23
|
+
});
|
|
@@ -157,9 +157,8 @@ export function shouldFailKnownFailureIssueValidation(result) {
|
|
|
157
157
|
return (result.summary?.errors || 0) > 0;
|
|
158
158
|
}
|
|
159
159
|
|
|
160
|
-
export function
|
|
160
|
+
export function buildKnownFailureIssueValidationSummaryParts(result) {
|
|
161
161
|
if (!result) return [];
|
|
162
|
-
|
|
163
162
|
const parts = [];
|
|
164
163
|
const byCode = result.summary?.byCode || {};
|
|
165
164
|
if (byCode.closed_but_failing) {
|
|
@@ -190,6 +189,11 @@ export function buildKnownFailureIssueValidationSummaryLines(result) {
|
|
|
190
189
|
parts.push("used stale GitHub cache");
|
|
191
190
|
}
|
|
192
191
|
|
|
192
|
+
return parts;
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
export function buildKnownFailureIssueValidationSummaryLines(result) {
|
|
196
|
+
const parts = buildKnownFailureIssueValidationSummaryParts(result);
|
|
193
197
|
if (parts.length === 0) return [];
|
|
194
198
|
|
|
195
199
|
return [
|
|
@@ -1,4 +1,8 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { findMatchingKnownFailureEntries } from "../known-failures/index.mjs";
|
|
2
|
+
import {
|
|
3
|
+
buildKnownFailureIssueValidationSummaryLines,
|
|
4
|
+
buildKnownFailureIssueValidationSummaryParts,
|
|
5
|
+
} from "../known-failures/github.mjs";
|
|
2
6
|
|
|
3
7
|
export function formatDuration(durationMs) {
|
|
4
8
|
const totalSeconds = Math.max(0, Math.round(durationMs / 1000));
|
|
@@ -52,48 +56,49 @@ export function buildRunSummaryLines(results, durationMs, knownFailureIssueValid
|
|
|
52
56
|
return buildCompactRunSummaryLines(results, durationMs, knownFailureIssueValidation);
|
|
53
57
|
}
|
|
54
58
|
|
|
59
|
+
export function buildRunSummaryData(results, durationMs, knownFailureIssueValidation = null) {
|
|
60
|
+
const totals = summarizeResults(results);
|
|
61
|
+
const serviceErrors = collectServiceErrors(results);
|
|
62
|
+
const knownFailureParts = buildKnownFailureIssueValidationSummaryParts(knownFailureIssueValidation);
|
|
63
|
+
return {
|
|
64
|
+
result: totals.failedServices > 0 ? "FAILED" : "PASSED",
|
|
65
|
+
totalServices: totals.totalServices,
|
|
66
|
+
failedServices: totals.failedServices,
|
|
67
|
+
passed: totals.passedFiles,
|
|
68
|
+
failed: totals.failedFiles,
|
|
69
|
+
skipped: totals.skippedFiles,
|
|
70
|
+
notRun: totals.notRunFiles,
|
|
71
|
+
files: totals.totalFiles,
|
|
72
|
+
duration: formatDuration(durationMs),
|
|
73
|
+
serviceErrors: serviceErrors.length,
|
|
74
|
+
knownFailureIssues: knownFailureParts,
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
|
|
55
78
|
export function buildCompactRunSummaryLines(
|
|
56
79
|
results,
|
|
57
80
|
durationMs,
|
|
58
81
|
knownFailureIssueValidation = null
|
|
59
82
|
) {
|
|
60
|
-
const
|
|
83
|
+
const summary = buildRunSummaryData(results, durationMs, knownFailureIssueValidation);
|
|
61
84
|
const lines = [
|
|
62
85
|
"",
|
|
63
|
-
`Summary: ${
|
|
86
|
+
`Summary: ${summary.passed} passed, ${summary.failed} failed, ${summary.skipped} skipped, ${summary.notRun} not run across ${summary.files} ${pluralize(summary.files, "file", "files")} in ${summary.duration}`,
|
|
64
87
|
];
|
|
65
88
|
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
lines.push("", "Failures:");
|
|
69
|
-
for (const failure of failures) {
|
|
70
|
-
lines.push(` ${failure.file.path}`);
|
|
71
|
-
lines.push(` ${failure.primaryMessage}`);
|
|
72
|
-
for (const detail of failure.extraLines.slice(0, 3)) {
|
|
73
|
-
lines.push(` ${detail}`);
|
|
74
|
-
}
|
|
75
|
-
}
|
|
76
|
-
}
|
|
77
|
-
|
|
78
|
-
const serviceErrors = collectServiceErrors(results);
|
|
79
|
-
if (serviceErrors.length > 0) {
|
|
80
|
-
lines.push("", "Runtime Errors:");
|
|
81
|
-
for (const item of serviceErrors) {
|
|
82
|
-
lines.push(` ${item.service}`);
|
|
83
|
-
lines.push(` ${item.message}`);
|
|
84
|
-
}
|
|
89
|
+
if (summary.serviceErrors > 0) {
|
|
90
|
+
lines.push(`Runtime errors: ${summary.serviceErrors}`);
|
|
85
91
|
}
|
|
86
92
|
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
);
|
|
90
|
-
if (knownFailureIssueLines.length > 0) {
|
|
91
|
-
lines.push(...knownFailureIssueLines);
|
|
93
|
+
if (summary.knownFailureIssues.length > 0) {
|
|
94
|
+
lines.push(`Known-failure issues: ${summary.knownFailureIssues.join(", ")}`);
|
|
92
95
|
}
|
|
93
96
|
|
|
94
97
|
lines.push("");
|
|
95
98
|
lines.push(
|
|
96
|
-
|
|
99
|
+
summary.result === "FAILED"
|
|
100
|
+
? `Result: FAILED (${summary.failedServices}/${summary.totalServices} services failed)`
|
|
101
|
+
: "Result: PASSED"
|
|
97
102
|
);
|
|
98
103
|
return lines;
|
|
99
104
|
}
|
|
@@ -183,6 +188,43 @@ export function buildDebugRunSummaryLines(results, durationMs, knownFailureIssue
|
|
|
183
188
|
return lines;
|
|
184
189
|
}
|
|
185
190
|
|
|
191
|
+
export function buildFailurePresentation(fileSummary, knownFailures = null) {
|
|
192
|
+
const rankedDetails = rankFailureDetails(fileSummary.failureDetails || []);
|
|
193
|
+
const primaryDetail = rankedDetails[0] || null;
|
|
194
|
+
const fallbackMessages = rankedDetails
|
|
195
|
+
.map((detail) => detail.message || detail.title)
|
|
196
|
+
.filter(Boolean)
|
|
197
|
+
.map((message) => sanitizeErrorMessage(String(message).trim()));
|
|
198
|
+
const details = [];
|
|
199
|
+
|
|
200
|
+
const responseLine = formatFailureResponsePreview(primaryDetail);
|
|
201
|
+
if (responseLine) details.push(responseLine);
|
|
202
|
+
|
|
203
|
+
const triageLine = formatInlineKnownFailureLine(fileSummary, knownFailures);
|
|
204
|
+
if (triageLine) details.push(triageLine);
|
|
205
|
+
|
|
206
|
+
const requestLine = formatFailureRequestHint(primaryDetail);
|
|
207
|
+
if (requestLine) details.push(requestLine);
|
|
208
|
+
|
|
209
|
+
for (const detail of rankedDetails.slice(primaryDetail ? 1 : 0)) {
|
|
210
|
+
const line = sanitizeInline(String(detail.message || detail.title || ""), 220);
|
|
211
|
+
if (line && !details.includes(line)) details.push(line);
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
return {
|
|
215
|
+
primary: sanitizeInline(
|
|
216
|
+
resolvePrimaryFailureMessage(
|
|
217
|
+
fileSummary,
|
|
218
|
+
{ error: fileSummary.suiteError || null },
|
|
219
|
+
primaryDetail,
|
|
220
|
+
fallbackMessages
|
|
221
|
+
),
|
|
222
|
+
240
|
|
223
|
+
),
|
|
224
|
+
details,
|
|
225
|
+
};
|
|
226
|
+
}
|
|
227
|
+
|
|
186
228
|
function sanitizeErrorMessage(message) {
|
|
187
229
|
return message
|
|
188
230
|
.replace(/Command failed with exit code (\d+): .*?[\\/]vendor[\\/]k6 run\b/g, "Default runtime failed with exit code $1:")
|
|
@@ -203,46 +245,6 @@ function summarizeResults(results) {
|
|
|
203
245
|
};
|
|
204
246
|
}
|
|
205
247
|
|
|
206
|
-
function collectFailedFiles(results) {
|
|
207
|
-
const failures = [];
|
|
208
|
-
for (const result of results) {
|
|
209
|
-
for (const suite of result.suites || []) {
|
|
210
|
-
for (const file of suite.files || []) {
|
|
211
|
-
if (file.status !== "failed") continue;
|
|
212
|
-
const rankedDetails = rankFailureDetails(file.failureDetails || []);
|
|
213
|
-
const primaryDetail = rankedDetails[0] || null;
|
|
214
|
-
const fallbackMessages = rankedDetails
|
|
215
|
-
.map((detail) => detail.message || detail.title)
|
|
216
|
-
.filter(Boolean)
|
|
217
|
-
.map((message) => sanitizeErrorMessage(String(message).trim()));
|
|
218
|
-
const extraLines = [];
|
|
219
|
-
if (primaryDetail) {
|
|
220
|
-
const responseLine = formatFailureResponsePreview(primaryDetail);
|
|
221
|
-
if (responseLine) extraLines.push(responseLine);
|
|
222
|
-
const triageLine = formatTriageLine(file.triage || null);
|
|
223
|
-
if (triageLine) extraLines.push(triageLine);
|
|
224
|
-
const requestLine = formatFailureRequestHint(primaryDetail);
|
|
225
|
-
if (requestLine) extraLines.push(requestLine);
|
|
226
|
-
} else {
|
|
227
|
-
const triageLine = formatTriageLine(file.triage || null);
|
|
228
|
-
if (triageLine) extraLines.push(triageLine);
|
|
229
|
-
}
|
|
230
|
-
|
|
231
|
-
for (const detail of rankedDetails.slice(primaryDetail ? 1 : 0)) {
|
|
232
|
-
const line = sanitizeErrorMessage(String(detail.message || detail.title || "").trim());
|
|
233
|
-
if (line && !extraLines.includes(line)) extraLines.push(line);
|
|
234
|
-
}
|
|
235
|
-
failures.push({
|
|
236
|
-
file,
|
|
237
|
-
primaryMessage: resolvePrimaryFailureMessage(file, suite, primaryDetail, fallbackMessages),
|
|
238
|
-
extraLines,
|
|
239
|
-
});
|
|
240
|
-
}
|
|
241
|
-
}
|
|
242
|
-
}
|
|
243
|
-
return failures.sort((left, right) => left.file.path.localeCompare(right.file.path));
|
|
244
|
-
}
|
|
245
|
-
|
|
246
248
|
function resolvePrimaryFailureMessage(file, suite, primaryDetail, fallbackMessages) {
|
|
247
249
|
if (primaryDetail?.message) {
|
|
248
250
|
return sanitizeErrorMessage(String(primaryDetail.message).trim());
|
|
@@ -295,32 +297,17 @@ function formatFailureRequestHint(detail) {
|
|
|
295
297
|
return `request: ${method} ${path}`;
|
|
296
298
|
}
|
|
297
299
|
|
|
298
|
-
function
|
|
299
|
-
if (!
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
const entry = triage.entries?.[0];
|
|
303
|
-
if (!entry?.issue) {
|
|
304
|
-
if (triage.status === "validation_unavailable") {
|
|
305
|
-
const reason = triage.availability?.reason ? ` (${triage.availability.reason})` : "";
|
|
306
|
-
return `triage: validation unavailable${reason}`;
|
|
307
|
-
}
|
|
308
|
-
return null;
|
|
309
|
-
}
|
|
300
|
+
function formatInlineKnownFailureLine(fileSummary, knownFailures) {
|
|
301
|
+
if (!knownFailures) return "triage: untriaged";
|
|
302
|
+
const matches = findMatchingKnownFailureEntries(knownFailures, fileSummary);
|
|
303
|
+
if (matches.length === 0) return "triage: untriaged";
|
|
310
304
|
|
|
305
|
+
const entry = matches[0];
|
|
311
306
|
const issueLabel = `#${entry.issue.number}`;
|
|
312
|
-
|
|
313
|
-
if (validationStatus === "closed_but_failing") {
|
|
314
|
-
return `triage: known issue ${issueLabel} closed but still failing`;
|
|
315
|
-
}
|
|
316
|
-
if (validationStatus === "validation_unavailable") {
|
|
317
|
-
const reason = triage.availability?.mode === "cache" ? "cache" : "validation unavailable";
|
|
318
|
-
return `triage: known issue ${issueLabel} (${reason})`;
|
|
319
|
-
}
|
|
320
|
-
if (entry.github?.state === "open" || entry.state === "open") {
|
|
307
|
+
if (entry.state === "open") {
|
|
321
308
|
return `triage: known issue ${issueLabel} open`;
|
|
322
309
|
}
|
|
323
|
-
if (entry.
|
|
310
|
+
if (entry.state === "closed") {
|
|
324
311
|
return `triage: known issue ${issueLabel} closed`;
|
|
325
312
|
}
|
|
326
313
|
return `triage: known issue ${issueLabel}`;
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import { describe, expect, it } from "vitest";
|
|
2
2
|
import {
|
|
3
|
+
buildFailurePresentation,
|
|
4
|
+
buildRunSummaryData,
|
|
3
5
|
buildRunSummaryLines,
|
|
4
6
|
formatDuration,
|
|
5
7
|
formatError,
|
|
@@ -61,8 +63,8 @@ describe("runner formatting", () => {
|
|
|
61
63
|
expect(longestServiceName([{ name: "api" }, { name: "frontend" }])).toBe(8);
|
|
62
64
|
});
|
|
63
65
|
|
|
64
|
-
it("builds
|
|
65
|
-
const
|
|
66
|
+
it("builds aggregate summary data without failure dumps", () => {
|
|
67
|
+
const summary = buildRunSummaryData(
|
|
66
68
|
[
|
|
67
69
|
{
|
|
68
70
|
name: "frontend",
|
|
@@ -74,96 +76,42 @@ describe("runner formatting", () => {
|
|
|
74
76
|
failedSuiteCount: 1,
|
|
75
77
|
totalFileCount: 3,
|
|
76
78
|
passedFileCount: 2,
|
|
79
|
+
failedFileCount: 1,
|
|
77
80
|
skippedFileCount: 0,
|
|
78
81
|
notRunFileCount: 0,
|
|
79
82
|
durationMs: 20_000,
|
|
80
|
-
suites: [
|
|
81
|
-
{
|
|
82
|
-
failed: true,
|
|
83
|
-
type: "e2e",
|
|
84
|
-
name: "auth",
|
|
85
|
-
framework: "playwright",
|
|
86
|
-
failedFiles: ["a.pw.testkit.ts"],
|
|
87
|
-
durationMs: 12_000,
|
|
88
|
-
error: "boom",
|
|
89
|
-
},
|
|
90
|
-
],
|
|
91
|
-
errors: ["worker broke"],
|
|
92
|
-
},
|
|
93
|
-
],
|
|
94
|
-
20_000
|
|
95
|
-
);
|
|
96
|
-
|
|
97
|
-
expect(lines.join("\n")).toContain("Summary: 2 passed, 0 failed, 0 skipped, 0 not run across 3 files");
|
|
98
|
-
expect(lines.join("\n")).toContain("Runtime Errors:");
|
|
99
|
-
expect(lines.join("\n")).toContain("worker broke");
|
|
100
|
-
expect(lines.at(-1)).toBe("Result: FAILED (1/1 services failed)");
|
|
101
|
-
});
|
|
102
|
-
|
|
103
|
-
it("marks services with only skipped suites as SKIP", () => {
|
|
104
|
-
const lines = buildRunSummaryLines(
|
|
105
|
-
[
|
|
106
|
-
{
|
|
107
|
-
name: "api",
|
|
108
|
-
skipped: false,
|
|
109
|
-
failed: false,
|
|
110
|
-
suiteCount: 1,
|
|
111
|
-
completedSuiteCount: 1,
|
|
112
|
-
skippedSuiteCount: 1,
|
|
113
|
-
failedSuiteCount: 0,
|
|
114
|
-
totalFileCount: 1,
|
|
115
|
-
passedFileCount: 0,
|
|
116
|
-
skippedFileCount: 1,
|
|
117
|
-
notRunFileCount: 0,
|
|
118
|
-
durationMs: 0,
|
|
119
|
-
suites: [],
|
|
120
|
-
errors: [],
|
|
121
|
-
},
|
|
122
|
-
],
|
|
123
|
-
0
|
|
124
|
-
);
|
|
125
|
-
|
|
126
|
-
expect(lines.join("\n")).toContain("Summary: 0 passed, 0 failed, 1 skipped, 0 not run across 1 file");
|
|
127
|
-
expect(lines.at(-1)).toBe("Result: PASSED");
|
|
128
|
-
});
|
|
129
|
-
|
|
130
|
-
it("appends known-failure issue validation summary lines", () => {
|
|
131
|
-
const lines = buildRunSummaryLines(
|
|
132
|
-
[
|
|
133
|
-
{
|
|
134
|
-
name: "api",
|
|
135
|
-
skipped: false,
|
|
136
|
-
failed: false,
|
|
137
|
-
suiteCount: 1,
|
|
138
|
-
completedSuiteCount: 1,
|
|
139
|
-
skippedSuiteCount: 0,
|
|
140
|
-
failedSuiteCount: 0,
|
|
141
|
-
totalFileCount: 1,
|
|
142
|
-
passedFileCount: 1,
|
|
143
|
-
skippedFileCount: 0,
|
|
144
|
-
notRunFileCount: 0,
|
|
145
|
-
durationMs: 1_000,
|
|
146
83
|
suites: [],
|
|
147
|
-
errors: [],
|
|
84
|
+
errors: ["worker broke"],
|
|
148
85
|
},
|
|
149
86
|
],
|
|
150
|
-
|
|
87
|
+
20_000,
|
|
151
88
|
{
|
|
152
89
|
summary: {
|
|
153
90
|
byCode: {
|
|
154
91
|
closed_but_failing: 2,
|
|
155
|
-
|
|
92
|
+
state_mismatch: 1,
|
|
156
93
|
},
|
|
157
94
|
},
|
|
158
95
|
}
|
|
159
96
|
);
|
|
160
97
|
|
|
161
|
-
expect(
|
|
162
|
-
|
|
163
|
-
|
|
98
|
+
expect(summary).toMatchObject({
|
|
99
|
+
result: "FAILED",
|
|
100
|
+
passed: 2,
|
|
101
|
+
failed: 1,
|
|
102
|
+
skipped: 0,
|
|
103
|
+
notRun: 0,
|
|
104
|
+
files: 3,
|
|
105
|
+
duration: "20s",
|
|
106
|
+
serviceErrors: 1,
|
|
107
|
+
});
|
|
108
|
+
expect(summary.knownFailureIssues).toEqual([
|
|
109
|
+
"2 closed issues still failing",
|
|
110
|
+
"1 state mismatch",
|
|
111
|
+
]);
|
|
164
112
|
});
|
|
165
113
|
|
|
166
|
-
it("
|
|
114
|
+
it("builds compact summary lines without failure and runtime dump sections", () => {
|
|
167
115
|
const lines = buildRunSummaryLines(
|
|
168
116
|
[
|
|
169
117
|
{
|
|
@@ -180,63 +128,77 @@ describe("runner formatting", () => {
|
|
|
180
128
|
skippedFileCount: 0,
|
|
181
129
|
notRunFileCount: 0,
|
|
182
130
|
durationMs: 500,
|
|
183
|
-
suites: [
|
|
184
|
-
|
|
185
|
-
failed: true,
|
|
186
|
-
type: "int",
|
|
187
|
-
name: "default",
|
|
188
|
-
framework: "k6",
|
|
189
|
-
failedFiles: ["__testkit__/health/health.int.testkit.ts"],
|
|
190
|
-
durationMs: 500,
|
|
191
|
-
files: [
|
|
192
|
-
{
|
|
193
|
-
path: "__testkit__/health/health.int.testkit.ts",
|
|
194
|
-
status: "failed",
|
|
195
|
-
error: "Default runtime thresholds failed: checks(rate==1.0)",
|
|
196
|
-
failureDetails: [
|
|
197
|
-
{
|
|
198
|
-
kind: "http-assertion",
|
|
199
|
-
key: "GET /health > status is 200",
|
|
200
|
-
title: "status is 200",
|
|
201
|
-
message: "GET /health expected 200, got 404",
|
|
202
|
-
request: {
|
|
203
|
-
method: "GET",
|
|
204
|
-
path: "/health",
|
|
205
|
-
requestId: "req-1",
|
|
206
|
-
},
|
|
207
|
-
response: {
|
|
208
|
-
status: 404,
|
|
209
|
-
bodyPreview: '{"error":"nope"}',
|
|
210
|
-
},
|
|
211
|
-
},
|
|
212
|
-
],
|
|
213
|
-
triage: {
|
|
214
|
-
status: "known_failure",
|
|
215
|
-
entries: [
|
|
216
|
-
{
|
|
217
|
-
id: "health-is-bad",
|
|
218
|
-
state: "open",
|
|
219
|
-
issue: {
|
|
220
|
-
repo: "acme/example",
|
|
221
|
-
number: 42,
|
|
222
|
-
},
|
|
223
|
-
},
|
|
224
|
-
],
|
|
225
|
-
},
|
|
226
|
-
},
|
|
227
|
-
],
|
|
228
|
-
error: null,
|
|
229
|
-
},
|
|
230
|
-
],
|
|
231
|
-
errors: [],
|
|
131
|
+
suites: [],
|
|
132
|
+
errors: ["worker broke"],
|
|
232
133
|
},
|
|
233
134
|
],
|
|
234
135
|
500
|
|
235
136
|
);
|
|
236
137
|
|
|
237
|
-
expect(lines.join("\n")).toContain("
|
|
238
|
-
expect(lines.join("\n")).toContain(
|
|
239
|
-
expect(lines.join("\n")).toContain("
|
|
240
|
-
expect(lines.join("\n")).not.toContain("
|
|
138
|
+
expect(lines.join("\n")).toContain("Summary: 0 passed, 1 failed, 0 skipped, 0 not run across 1 file");
|
|
139
|
+
expect(lines.join("\n")).toContain("Runtime errors: 1");
|
|
140
|
+
expect(lines.join("\n")).not.toContain("Failures:");
|
|
141
|
+
expect(lines.join("\n")).not.toContain("Runtime Errors:");
|
|
142
|
+
});
|
|
143
|
+
|
|
144
|
+
it("builds inline failure presentations from structured HTTP assertion details", () => {
|
|
145
|
+
const presentation = buildFailurePresentation(
|
|
146
|
+
{
|
|
147
|
+
service: "api",
|
|
148
|
+
type: "int",
|
|
149
|
+
path: "__testkit__/health/health.int.testkit.ts",
|
|
150
|
+
error: "Default runtime thresholds failed: checks(rate==1.0)",
|
|
151
|
+
failureDetails: [
|
|
152
|
+
{
|
|
153
|
+
kind: "http-assertion",
|
|
154
|
+
key: "GET /health > status is 200",
|
|
155
|
+
title: "status is 200",
|
|
156
|
+
message: "GET /health expected 200, got 404",
|
|
157
|
+
request: {
|
|
158
|
+
method: "GET",
|
|
159
|
+
path: "/health",
|
|
160
|
+
requestId: "req-1",
|
|
161
|
+
},
|
|
162
|
+
response: {
|
|
163
|
+
status: 404,
|
|
164
|
+
bodyPreview: '{"error":"nope"}',
|
|
165
|
+
},
|
|
166
|
+
},
|
|
167
|
+
],
|
|
168
|
+
},
|
|
169
|
+
{
|
|
170
|
+
schemaVersion: 1,
|
|
171
|
+
issueRepo: "acme/example",
|
|
172
|
+
entries: [
|
|
173
|
+
{
|
|
174
|
+
id: "health-is-bad",
|
|
175
|
+
title: "Health is bad",
|
|
176
|
+
classification: "product_bug",
|
|
177
|
+
state: "open",
|
|
178
|
+
issue: {
|
|
179
|
+
repo: "acme/example",
|
|
180
|
+
number: 42,
|
|
181
|
+
url: "https://github.com/acme/example/issues/42",
|
|
182
|
+
},
|
|
183
|
+
description: "desc",
|
|
184
|
+
whyFailing: "because",
|
|
185
|
+
lastReviewedAt: "2026-01-01",
|
|
186
|
+
matches: [
|
|
187
|
+
{
|
|
188
|
+
service: "api",
|
|
189
|
+
type: "int",
|
|
190
|
+
path: "__testkit__/health/health.int.testkit.ts",
|
|
191
|
+
failureKey: "GET /health > status is 200",
|
|
192
|
+
},
|
|
193
|
+
],
|
|
194
|
+
},
|
|
195
|
+
],
|
|
196
|
+
}
|
|
197
|
+
);
|
|
198
|
+
|
|
199
|
+
expect(presentation.primary).toBe("GET /health expected 200, got 404");
|
|
200
|
+
expect(presentation.details).toContain('response: {"error":"nope"}');
|
|
201
|
+
expect(presentation.details).toContain("triage: known issue #42 open");
|
|
202
|
+
expect(presentation.details).toContain("logs: requestId=req-1");
|
|
241
203
|
});
|
|
242
204
|
});
|
|
@@ -61,6 +61,7 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
|
|
|
61
61
|
configs[0]?.testkit?.reporting || null
|
|
62
62
|
);
|
|
63
63
|
const reporter = opts.reporter || null;
|
|
64
|
+
reporter?.setKnownFailures?.(knownFailures);
|
|
64
65
|
const logRegistry = createRunLogRegistry(productDir);
|
|
65
66
|
const workerState = {
|
|
66
67
|
workerCount: 0,
|
|
@@ -121,6 +122,7 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
|
|
|
121
122
|
const timings = loadTimings(productDir);
|
|
122
123
|
const graphs = buildRuntimeGraphs(executedPlans);
|
|
123
124
|
const queue = buildTaskQueue(executedPlans, graphs, timings);
|
|
125
|
+
reporter?.setTotalFileCount?.(queue.length);
|
|
124
126
|
for (const task of queue) {
|
|
125
127
|
task.scenarioSeed = opts.scenarioSeed || null;
|
|
126
128
|
}
|
|
@@ -81,6 +81,7 @@ export async function runWorker(
|
|
|
81
81
|
} catch (error) {
|
|
82
82
|
const message = formatError(error);
|
|
83
83
|
errors.push(message);
|
|
84
|
+
reporter?.runtimeError?.(task, message);
|
|
84
85
|
recordGraphError(trackers, { targetNames: lease?.context?.targetNames || [task.targetName] }, message);
|
|
85
86
|
await runtimeManager.release(lease, {
|
|
86
87
|
invalidate: lease !== null,
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@elench/testkit-bridge",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.80",
|
|
4
4
|
"description": "Browser bridge helpers for testkit",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -22,7 +22,7 @@
|
|
|
22
22
|
"typecheck": "tsc -p tsconfig.json --noEmit"
|
|
23
23
|
},
|
|
24
24
|
"dependencies": {
|
|
25
|
-
"@elench/testkit-protocol": "0.1.
|
|
25
|
+
"@elench/testkit-protocol": "0.1.80"
|
|
26
26
|
},
|
|
27
27
|
"private": false
|
|
28
28
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@elench/testkit",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.80",
|
|
4
4
|
"description": "CLI for discovering and running local HTTP, DAL, and Playwright test suites",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"workspaces": [
|
|
@@ -81,17 +81,21 @@
|
|
|
81
81
|
},
|
|
82
82
|
"dependencies": {
|
|
83
83
|
"@babel/code-frame": "^7.29.0",
|
|
84
|
-
"@elench/next-analysis": "0.1.
|
|
85
|
-
"@elench/testkit-bridge": "0.1.
|
|
86
|
-
"@elench/testkit-protocol": "0.1.
|
|
87
|
-
"@elench/ts-analysis": "0.1.
|
|
84
|
+
"@elench/next-analysis": "0.1.80",
|
|
85
|
+
"@elench/testkit-bridge": "0.1.80",
|
|
86
|
+
"@elench/testkit-protocol": "0.1.80",
|
|
87
|
+
"@elench/ts-analysis": "0.1.80",
|
|
88
88
|
"@oclif/core": "^4.10.6",
|
|
89
89
|
"esbuild": "^0.25.11",
|
|
90
90
|
"execa": "^9.5.0",
|
|
91
|
+
"figures": "^6.1.0",
|
|
91
92
|
"ink": "^7.0.1",
|
|
92
93
|
"picocolors": "^1.1.1",
|
|
93
94
|
"react": "^19.2.5",
|
|
94
|
-
"
|
|
95
|
+
"string-width": "^8.1.0",
|
|
96
|
+
"strip-ansi": "^7.1.2",
|
|
97
|
+
"typescript": "^5.9.3",
|
|
98
|
+
"wrap-ansi": "^10.0.0"
|
|
95
99
|
},
|
|
96
100
|
"engines": {
|
|
97
101
|
"node": ">=18"
|