@restormel/testing-report 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/dist/artifact-filenames.d.ts +11 -0
- package/dist/artifact-filenames.d.ts.map +1 -0
- package/dist/artifact-filenames.js +11 -0
- package/dist/artifact-filenames.js.map +1 -0
- package/dist/failure-bucket.d.ts +10 -0
- package/dist/failure-bucket.d.ts.map +1 -0
- package/dist/failure-bucket.js +35 -0
- package/dist/failure-bucket.js.map +1 -0
- package/dist/github-summary.d.ts +20 -0
- package/dist/github-summary.d.ts.map +1 -0
- package/dist/github-summary.js +86 -0
- package/dist/github-summary.js.map +1 -0
- package/dist/index.d.ts +14 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +13 -0
- package/dist/index.js.map +1 -0
- package/dist/junit.d.ts +6 -0
- package/dist/junit.d.ts.map +1 -0
- package/dist/junit.js +54 -0
- package/dist/junit.js.map +1 -0
- package/dist/markdown-summary.d.ts +13 -0
- package/dist/markdown-summary.d.ts.map +1 -0
- package/dist/markdown-summary.js +113 -0
- package/dist/markdown-summary.js.map +1 -0
- package/dist/mvp-json-report.d.ts +58 -0
- package/dist/mvp-json-report.d.ts.map +1 -0
- package/dist/mvp-json-report.js +77 -0
- package/dist/mvp-json-report.js.map +1 -0
- package/dist/read-artifacts.d.ts +17 -0
- package/dist/read-artifacts.d.ts.map +1 -0
- package/dist/read-artifacts.js +63 -0
- package/dist/read-artifacts.js.map +1 -0
- package/dist/terminal-summary.d.ts +7 -0
- package/dist/terminal-summary.d.ts.map +1 -0
- package/dist/terminal-summary.js +40 -0
- package/dist/terminal-summary.js.map +1 -0
- package/dist/write-artifacts.d.ts +25 -0
- package/dist/write-artifacts.d.ts.map +1 -0
- package/dist/write-artifacts.js +49 -0
- package/dist/write-artifacts.js.map +1 -0
- package/package.json +43 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) Allotment Technology Ltd
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/** Stable artefact basenames under each run directory (early-adopter contract). */
|
|
2
|
+
export declare const RUN_JSON: "run.json";
|
|
3
|
+
export declare const TRACES_JSON: "traces.json";
|
|
4
|
+
export declare const WARNINGS_TXT: "warnings.txt";
|
|
5
|
+
export declare const REPORT_JSON: "report.json";
|
|
6
|
+
export declare const SUMMARY_MD: "summary.md";
|
|
7
|
+
export declare const GITHUB_SUMMARY_MD: "github-summary.md";
|
|
8
|
+
export declare const JUNIT_XML: "junit.xml";
|
|
9
|
+
/** Written when `run` exits before a {@link RunRecord} exists (config / suite resolution errors). */
|
|
10
|
+
export declare const PRE_RUN_FAILURE_JSON: "pre-run-failure.json";
|
|
11
|
+
//# sourceMappingURL=artifact-filenames.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"artifact-filenames.d.ts","sourceRoot":"","sources":["../src/artifact-filenames.ts"],"names":[],"mappings":"AAAA,mFAAmF;AACnF,eAAO,MAAM,QAAQ,EAAG,UAAmB,CAAC;AAC5C,eAAO,MAAM,WAAW,EAAG,aAAsB,CAAC;AAClD,eAAO,MAAM,YAAY,EAAG,cAAuB,CAAC;AACpD,eAAO,MAAM,WAAW,EAAG,aAAsB,CAAC;AAClD,eAAO,MAAM,UAAU,EAAG,YAAqB,CAAC;AAChD,eAAO,MAAM,iBAAiB,EAAG,mBAA4B,CAAC;AAC9D,eAAO,MAAM,SAAS,EAAG,WAAoB,CAAC;AAC9C,qGAAqG;AACrG,eAAO,MAAM,oBAAoB,EAAG,sBAA+B,CAAC"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/** Stable artefact basenames under each run directory (early-adopter contract). */
|
|
2
|
+
export const RUN_JSON = "run.json";
|
|
3
|
+
export const TRACES_JSON = "traces.json";
|
|
4
|
+
export const WARNINGS_TXT = "warnings.txt";
|
|
5
|
+
export const REPORT_JSON = "report.json";
|
|
6
|
+
export const SUMMARY_MD = "summary.md";
|
|
7
|
+
export const GITHUB_SUMMARY_MD = "github-summary.md";
|
|
8
|
+
export const JUNIT_XML = "junit.xml";
|
|
9
|
+
/** Written when `run` exits before a {@link RunRecord} exists (config / suite resolution errors). */
|
|
10
|
+
export const PRE_RUN_FAILURE_JSON = "pre-run-failure.json";
|
|
11
|
+
//# sourceMappingURL=artifact-filenames.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"artifact-filenames.js","sourceRoot":"","sources":["../src/artifact-filenames.ts"],"names":[],"mappings":"AAAA,mFAAmF;AACnF,MAAM,CAAC,MAAM,QAAQ,GAAG,UAAmB,CAAC;AAC5C,MAAM,CAAC,MAAM,WAAW,GAAG,aAAsB,CAAC;AAClD,MAAM,CAAC,MAAM,YAAY,GAAG,cAAuB,CAAC;AACpD,MAAM,CAAC,MAAM,WAAW,GAAG,aAAsB,CAAC;AAClD,MAAM,CAAC,MAAM,UAAU,GAAG,YAAqB,CAAC;AAChD,MAAM,CAAC,MAAM,iBAAiB,GAAG,mBAA4B,CAAC;AAC9D,MAAM,CAAC,MAAM,SAAS,GAAG,WAAoB,CAAC;AAC9C,qGAAqG;AACrG,MAAM,CAAC,MAAM,oBAAoB,GAAG,sBAA+B,CAAC"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { Verdict } from "@restormel/testing-core";
|
|
2
|
+
/**
|
|
3
|
+
* Coarse triage category derived from `reasonCode` (stable for dashboards).
|
|
4
|
+
*/
|
|
5
|
+
export type FailureBucket = "none" | "assertion" | "timeout" | "adapter" | "browser" | "judge" | "keys_config" | "unsupported" | "config" | "unknown";
|
|
6
|
+
/**
|
|
7
|
+
* Map runner reason codes to a failure bucket for triage (no model internals).
|
|
8
|
+
*/
|
|
9
|
+
export declare function inferFailureBucket(reasonCode: string, verdict: Verdict): FailureBucket;
|
|
10
|
+
//# sourceMappingURL=failure-bucket.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"failure-bucket.d.ts","sourceRoot":"","sources":["../src/failure-bucket.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAEvD;;GAEG;AACH,MAAM,MAAM,aAAa,GACrB,MAAM,GACN,WAAW,GACX,SAAS,GACT,SAAS,GACT,SAAS,GACT,OAAO,GACP,aAAa,GACb,aAAa,GACb,QAAQ,GACR,SAAS,CAAC;AAEd;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,GAAG,aAAa,CAiCtF"}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Map runner reason codes to a failure bucket for triage (no model internals).
|
|
3
|
+
*/
|
|
4
|
+
export function inferFailureBucket(reasonCode, verdict) {
|
|
5
|
+
if (verdict === "passed") {
|
|
6
|
+
return "none";
|
|
7
|
+
}
|
|
8
|
+
const c = reasonCode.toUpperCase();
|
|
9
|
+
if (c.startsWith("URL_") || c.startsWith("TEXT_") || c.startsWith("DOM_") || c === "STRUCTURED_MISMATCH") {
|
|
10
|
+
return "assertion";
|
|
11
|
+
}
|
|
12
|
+
if (c === "TIMEOUT") {
|
|
13
|
+
return "timeout";
|
|
14
|
+
}
|
|
15
|
+
if (c === "ADAPTER_ERROR") {
|
|
16
|
+
return "adapter";
|
|
17
|
+
}
|
|
18
|
+
if (c === "BROWSER_ERROR") {
|
|
19
|
+
return "browser";
|
|
20
|
+
}
|
|
21
|
+
if (c.startsWith("JUDGE_")) {
|
|
22
|
+
if (c === "JUDGE_NO_MODEL") {
|
|
23
|
+
return "keys_config";
|
|
24
|
+
}
|
|
25
|
+
return "judge";
|
|
26
|
+
}
|
|
27
|
+
if (c === "GOAL_TYPE_UNSUPPORTED" || c === "STRUCTURED_PATH_UNKNOWN" || c === "NO_CRITERIA") {
|
|
28
|
+
return "unsupported";
|
|
29
|
+
}
|
|
30
|
+
if (c.includes("CONFIG") || c === "INVALID") {
|
|
31
|
+
return "config";
|
|
32
|
+
}
|
|
33
|
+
return "unknown";
|
|
34
|
+
}
|
|
35
|
+
//# sourceMappingURL=failure-bucket.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"failure-bucket.js","sourceRoot":"","sources":["../src/failure-bucket.ts"],"names":[],"mappings":"AAiBA;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAAC,UAAkB,EAAE,OAAgB;IACrE,IAAI,OAAO,KAAK,QAAQ,EAAE,CAAC;QACzB,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,MAAM,CAAC,GAAG,UAAU,CAAC,WAAW,EAAE,CAAC;IAEnC,IAAI,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,qBAAqB,EAAE,CAAC;QACzG,OAAO,WAAW,CAAC;IACrB,CAAC;IACD,IAAI,CAAC,KAAK,SAAS,EAAE,CAAC;QACpB,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,IAAI,CAAC,KAAK,eAAe,EAAE,CAAC;QAC1B,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,IAAI,CAAC,KAAK,eAAe,EAAE,CAAC;QAC1B,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,IAAI,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC3B,IAAI,CAAC,KAAK,gBAAgB,EAAE,CAAC;YAC3B,OAAO,aAAa,CAAC;QACvB,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IACD,IAAI,CAAC,KAAK,uBAAuB,IAAI,CAAC,KAAK,yBAAyB,IAAI,CAAC,KAAK,aAAa,EAAE,CAAC;QAC5F,OAAO,aAAa,CAAC;IACvB,CAAC;IACD,IAAI,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,SAAS,EAAE,CAAC;QAC5C,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC"}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import type { RunRecord, SuiteReportSlice } from "@restormel/testing-core";
|
|
2
|
+
export interface GithubSummaryContext {
|
|
3
|
+
run: RunRecord;
|
|
4
|
+
suite?: SuiteReportSlice;
|
|
5
|
+
warnings?: string[];
|
|
6
|
+
/** Shown under the title (e.g. PR, target URL from CI). */
|
|
7
|
+
ciContext?: {
|
|
8
|
+
prNumber?: string;
|
|
9
|
+
targetUrl?: string;
|
|
10
|
+
};
|
|
11
|
+
/** Single-line bash snippet for triage. */
|
|
12
|
+
reproduceCommand?: string;
|
|
13
|
+
/** Run artefact directory (for upload-artifact hints). */
|
|
14
|
+
artifactDirHint?: string;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Markdown tuned for `$GITHUB_STEP_SUMMARY`: short, scannable, actionable.
|
|
18
|
+
*/
|
|
19
|
+
export declare function buildGithubStepSummaryMarkdown(ctx: GithubSummaryContext): string;
|
|
20
|
+
//# sourceMappingURL=github-summary.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"github-summary.d.ts","sourceRoot":"","sources":["../src/github-summary.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAG3E,MAAM,WAAW,oBAAoB;IACnC,GAAG,EAAE,SAAS,CAAC;IACf,KAAK,CAAC,EAAE,gBAAgB,CAAC;IACzB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,2DAA2D;IAC3D,SAAS,CAAC,EAAE;QAAE,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACtD,2CAA2C;IAC3C,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,0DAA0D;IAC1D,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED;;GAEG;AACH,wBAAgB,8BAA8B,CAAC,GAAG,EAAE,oBAAoB,GAAG,MAAM,CA0FhF"}
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import { inferFailureBucket } from "./failure-bucket.js";
|
|
2
|
+
/**
|
|
3
|
+
* Markdown tuned for `$GITHUB_STEP_SUMMARY`: short, scannable, actionable.
|
|
4
|
+
*/
|
|
5
|
+
export function buildGithubStepSummaryMarkdown(ctx) {
|
|
6
|
+
const { run, suite, warnings, ciContext, reproduceCommand, artifactDirHint } = ctx;
|
|
7
|
+
const lines = [];
|
|
8
|
+
lines.push(`## Restormel Testing`);
|
|
9
|
+
lines.push("");
|
|
10
|
+
if (ciContext?.prNumber !== undefined) {
|
|
11
|
+
lines.push(`**PR:** #${ciContext.prNumber}`);
|
|
12
|
+
}
|
|
13
|
+
lines.push(`**Verdict:** **${run.verdict}** · **Run** \`${run.id}\``);
|
|
14
|
+
lines.push(`**Suite** \`${run.suiteId}\` · **Env** \`${run.environmentId}\``);
|
|
15
|
+
if (ciContext?.targetUrl !== undefined) {
|
|
16
|
+
lines.push(`**Target URL:** \`${ciContext.targetUrl}\``);
|
|
17
|
+
}
|
|
18
|
+
lines.push(`**Commit:** \`${run.commitSha ?? "n/a"}\` · **Repository:** \`${run.repository ?? "n/a"}\``);
|
|
19
|
+
lines.push("");
|
|
20
|
+
lines.push(`*Execution: inline in this job (no hosted orchestration).*`);
|
|
21
|
+
lines.push("");
|
|
22
|
+
const failed = run.goalRuns.filter((g) => g.verdict !== "passed");
|
|
23
|
+
if (failed.length > 0) {
|
|
24
|
+
lines.push(`### Needs attention`);
|
|
25
|
+
lines.push("");
|
|
26
|
+
for (const g of failed) {
|
|
27
|
+
const bucket = inferFailureBucket(g.reasonCode, g.verdict);
|
|
28
|
+
lines.push(`- **\`${g.goalId}\`** — ${g.verdict} (**${bucket}**) — ${g.summary}`);
|
|
29
|
+
if (g.evidenceRefs.length > 0) {
|
|
30
|
+
lines.push(` - Evidence: \`${g.evidenceRefs.join("`, `")}\``);
|
|
31
|
+
}
|
|
32
|
+
if (g.retriesUsed > 0) {
|
|
33
|
+
lines.push(` - Retries used: ${g.retriesUsed}`);
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
lines.push("");
|
|
37
|
+
}
|
|
38
|
+
lines.push(`### All goals`);
|
|
39
|
+
lines.push("");
|
|
40
|
+
lines.push(`| Goal | Result | Bucket |`);
|
|
41
|
+
lines.push(`|------|--------|--------|`);
|
|
42
|
+
for (const g of run.goalRuns) {
|
|
43
|
+
const bucket = inferFailureBucket(g.reasonCode, g.verdict);
|
|
44
|
+
lines.push(`| ${g.goalId} | ${g.verdict} | ${bucket} |`);
|
|
45
|
+
}
|
|
46
|
+
lines.push("");
|
|
47
|
+
if (suite?.description) {
|
|
48
|
+
lines.push(`### Suite note`);
|
|
49
|
+
lines.push("");
|
|
50
|
+
lines.push(suite.description);
|
|
51
|
+
lines.push("");
|
|
52
|
+
}
|
|
53
|
+
if (run.keysModelMeta !== undefined && run.keysModelMeta.length > 0) {
|
|
54
|
+
lines.push(`### Models (Keys)`);
|
|
55
|
+
lines.push("");
|
|
56
|
+
for (const m of run.keysModelMeta) {
|
|
57
|
+
lines.push(`- \`${m.logicalRef}\` → ${m.provider ?? "?"}/${m.model ?? "?"} (${m.invocationCount ?? 0} calls, ${m.resolutionSource ?? "?"})`);
|
|
58
|
+
}
|
|
59
|
+
lines.push("");
|
|
60
|
+
}
|
|
61
|
+
if (warnings !== undefined && warnings.length > 0) {
|
|
62
|
+
lines.push(`### Warnings`);
|
|
63
|
+
lines.push("");
|
|
64
|
+
for (const w of warnings) {
|
|
65
|
+
lines.push(`- ${w}`);
|
|
66
|
+
}
|
|
67
|
+
lines.push("");
|
|
68
|
+
}
|
|
69
|
+
if (reproduceCommand !== undefined && reproduceCommand.length > 0) {
|
|
70
|
+
lines.push(`### Reproduce locally`);
|
|
71
|
+
lines.push("");
|
|
72
|
+
lines.push("```bash");
|
|
73
|
+
lines.push(reproduceCommand);
|
|
74
|
+
lines.push("```");
|
|
75
|
+
lines.push("");
|
|
76
|
+
}
|
|
77
|
+
if (artifactDirHint !== undefined && artifactDirHint.length > 0) {
|
|
78
|
+
lines.push(`Upload **${artifactDirHint}** with \`actions/upload-artifact\` for full outputs (screenshots, \`run.json\`, \`report.json\`, \`junit.xml\`, …).`);
|
|
79
|
+
lines.push("");
|
|
80
|
+
}
|
|
81
|
+
else {
|
|
82
|
+
lines.push(`_Artefacts include \`report.json\`, \`summary.md\`, \`junit.xml\`, traces, screenshots (if any)._`);
|
|
83
|
+
}
|
|
84
|
+
return lines.join("\n");
|
|
85
|
+
}
|
|
86
|
+
//# sourceMappingURL=github-summary.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"github-summary.js","sourceRoot":"","sources":["../src/github-summary.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AAczD;;GAEG;AACH,MAAM,UAAU,8BAA8B,CAAC,GAAyB;IACtE,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,gBAAgB,EAAE,eAAe,EAAE,GAAG,GAAG,CAAC;IACnF,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,KAAK,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;IACnC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,IAAI,SAAS,EAAE,QAAQ,KAAK,SAAS,EAAE,CAAC;QACtC,KAAK,CAAC,IAAI,CAAC,YAAY,SAAS,CAAC,QAAQ,EAAE,CAAC,CAAC;IAC/C,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,kBAAkB,GAAG,CAAC,OAAO,kBAAkB,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC;IACtE,KAAK,CAAC,IAAI,CAAC,eAAe,GAAG,CAAC,OAAO,kBAAkB,GAAG,CAAC,aAAa,IAAI,CAAC,CAAC;IAC9E,IAAI,SAAS,EAAE,SAAS,KAAK,SAAS,EAAE,CAAC;QACvC,KAAK,CAAC,IAAI,CAAC,qBAAqB,SAAS,CAAC,SAAS,IAAI,CAAC,CAAC;IAC3D,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,iBAAiB,GAAG,CAAC,SAAS,IAAI,KAAK,0BAA0B,GAAG,CAAC,UAAU,IAAI,KAAK,IAAI,CAAC,CAAC;IACzG,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,4DAA4D,CAAC,CAAC;IACzE,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,MAAM,MAAM,GAAG,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC;IAClE,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,KAAK,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;QAClC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;YACvB,MAAM,MAAM,GAAG,kBAAkB,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC;YAC3D,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,UAAU,CAAC,CAAC,OAAO,OAAO,MAAM,SAAS,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;YAClF,IAAI,CAAC,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC9B,KAAK,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YACjE,CAAC;YACD,IAAI,CAAC,CAAC,WAAW,GAAG,CAAC,EAAE,CAAC;gBACtB,KAAK,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;YACnD,CAAC;QACH,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;IAC5B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC;IACzC,KAAK,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC;IACzC,KAAK,MAAM,CAAC,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC;QAC7B,MAAM,MAAM,GAAG,kBAAkB,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC;QAC3D,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,MAAM,CAAC,CAAC,OAAO,MAAM,MAAM,IAAI,CAAC,CAAC;IAC3D,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,IAAI,KAAK,EAAE,WAAW,EAAE,CAAC;QACvB,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAC7B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;QAC9B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,IAAI,GAAG,CAAC,aAAa,KAAK,SAAS,IAAI,GAAG,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACpE,KAAK,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;QAChC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,MAAM,CAAC,IAAI,GAAG,CAAC,aAAa,EAAE,CAAC;YAClC,KAAK,CAAC,IAAI,CACR,OAAO,CAAC,CAAC,UAAU,QAAQ,CAAC,CAAC,QAAQ,IAAI,GAAG,IAAI,CAAC,CAAC,KAAK,IAAI,GAAG,KAAK,CAAC,CAAC,eAAe,IAAI,CAAC,WAAW,CAAC,CAAC,gBAAgB,IAAI,GAAG,GAAG,CACjI,CAAC;QACJ,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,IAAI,QAAQ,KAAK,SAAS,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClD,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QAC3B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;YACzB,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACvB,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,IAAI,gBAAgB,KAAK,SAAS,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClE,KAAK,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;QACpC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACtB,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAC7B,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAClB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,IAAI,eAAe,KAAK,SAAS,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAChE,KAAK,CAAC,IAAI,CAAC,YAAY,eAAe,sHAAsH,CAAC,CAAC;QAC9J,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;SAAM,CAAC;QACN,KAAK,CAAC,IAAI,CAAC,mGAAmG,CAAC,CAAC;IAClH,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC"}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @restormel/testing-report — run artefacts, MVP reports, and summaries.
|
|
3
|
+
*/
|
|
4
|
+
export { GITHUB_SUMMARY_MD, JUNIT_XML, PRE_RUN_FAILURE_JSON, REPORT_JSON, RUN_JSON, SUMMARY_MD, TRACES_JSON, WARNINGS_TXT, } from "./artifact-filenames.js";
|
|
5
|
+
export { inferFailureBucket, type FailureBucket } from "./failure-bucket.js";
|
|
6
|
+
export { buildGithubStepSummaryMarkdown, type GithubSummaryContext } from "./github-summary.js";
|
|
7
|
+
export { buildJUnitXml } from "./junit.js";
|
|
8
|
+
export { buildMarkdownSummary, type MarkdownSummaryOptions } from "./markdown-summary.js";
|
|
9
|
+
export { buildMvpJsonReport, MVP_REPORT_SCHEMA_VERSION, serializeMvpJsonReport, type MvpGoalReportRow, type MvpJsonReportV1, } from "./mvp-json-report.js";
|
|
10
|
+
export { formatRunSummary, formatRunSummaryLines } from "./terminal-summary.js";
|
|
11
|
+
export { readRunArtifacts } from "./read-artifacts.js";
|
|
12
|
+
export type { LoadedRunArtifacts } from "./read-artifacts.js";
|
|
13
|
+
export { writeRunArtifacts, writeRunReportBundle, type RunArtifactPayload, type RunReportBundleInput, } from "./write-artifacts.js";
|
|
14
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,OAAO,EACL,iBAAiB,EACjB,SAAS,EACT,oBAAoB,EACpB,WAAW,EACX,QAAQ,EACR,UAAU,EACV,WAAW,EACX,YAAY,GACb,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAE,kBAAkB,EAAE,KAAK,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAC7E,OAAO,EAAE,8BAA8B,EAAE,KAAK,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAChG,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC3C,OAAO,EAAE,oBAAoB,EAAE,KAAK,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AAC1F,OAAO,EACL,kBAAkB,EAClB,yBAAyB,EACzB,sBAAsB,EACtB,KAAK,gBAAgB,EACrB,KAAK,eAAe,GACrB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,gBAAgB,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAChF,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AACvD,YAAY,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AAC9D,OAAO,EACL,iBAAiB,EACjB,oBAAoB,EACpB,KAAK,kBAAkB,EACvB,KAAK,oBAAoB,GAC1B,MAAM,sBAAsB,CAAC"}
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @restormel/testing-report — run artefacts, MVP reports, and summaries.
|
|
3
|
+
*/
|
|
4
|
+
export { GITHUB_SUMMARY_MD, JUNIT_XML, PRE_RUN_FAILURE_JSON, REPORT_JSON, RUN_JSON, SUMMARY_MD, TRACES_JSON, WARNINGS_TXT, } from "./artifact-filenames.js";
|
|
5
|
+
export { inferFailureBucket } from "./failure-bucket.js";
|
|
6
|
+
export { buildGithubStepSummaryMarkdown } from "./github-summary.js";
|
|
7
|
+
export { buildJUnitXml } from "./junit.js";
|
|
8
|
+
export { buildMarkdownSummary } from "./markdown-summary.js";
|
|
9
|
+
export { buildMvpJsonReport, MVP_REPORT_SCHEMA_VERSION, serializeMvpJsonReport, } from "./mvp-json-report.js";
|
|
10
|
+
export { formatRunSummary, formatRunSummaryLines } from "./terminal-summary.js";
|
|
11
|
+
export { readRunArtifacts } from "./read-artifacts.js";
|
|
12
|
+
export { writeRunArtifacts, writeRunReportBundle, } from "./write-artifacts.js";
|
|
13
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,OAAO,EACL,iBAAiB,EACjB,SAAS,EACT,oBAAoB,EACpB,WAAW,EACX,QAAQ,EACR,UAAU,EACV,WAAW,EACX,YAAY,GACb,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAE,kBAAkB,EAAsB,MAAM,qBAAqB,CAAC;AAC7E,OAAO,EAAE,8BAA8B,EAA6B,MAAM,qBAAqB,CAAC;AAChG,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC3C,OAAO,EAAE,oBAAoB,EAA+B,MAAM,uBAAuB,CAAC;AAC1F,OAAO,EACL,kBAAkB,EAClB,yBAAyB,EACzB,sBAAsB,GAGvB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,gBAAgB,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAChF,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAEvD,OAAO,EACL,iBAAiB,EACjB,oBAAoB,GAGrB,MAAM,sBAAsB,CAAC"}
|
package/dist/junit.d.ts
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import type { RunRecord } from "@restormel/testing-core";
|
|
2
|
+
/**
|
|
3
|
+
* Minimal JUnit XML: one `<testcase>` per goal. `failed` → `<failure>`, `indeterminate` → `<error>`.
|
|
4
|
+
*/
|
|
5
|
+
export declare function buildJUnitXml(run: RunRecord): string;
|
|
6
|
+
//# sourceMappingURL=junit.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"junit.d.ts","sourceRoot":"","sources":["../src/junit.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AAkBzD;;GAEG;AACH,wBAAgB,aAAa,CAAC,GAAG,EAAE,SAAS,GAAG,MAAM,CAkCpD"}
|
package/dist/junit.js
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
function xmlEscape(s) {
|
|
2
|
+
return s
|
|
3
|
+
.replace(/&/g, "&")
|
|
4
|
+
.replace(/</g, "<")
|
|
5
|
+
.replace(/>/g, ">")
|
|
6
|
+
.replace(/"/g, """);
|
|
7
|
+
}
|
|
8
|
+
function durationSeconds(run) {
|
|
9
|
+
if (!run.startedAt || !run.endedAt)
|
|
10
|
+
return 0;
|
|
11
|
+
const a = Date.parse(run.startedAt);
|
|
12
|
+
const b = Date.parse(run.endedAt);
|
|
13
|
+
if (!Number.isFinite(a) || !Number.isFinite(b) || b < a)
|
|
14
|
+
return 0;
|
|
15
|
+
return Math.max(0, (b - a) / 1000);
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* Minimal JUnit XML: one `<testcase>` per goal. `failed` → `<failure>`, `indeterminate` → `<error>`.
|
|
19
|
+
*/
|
|
20
|
+
export function buildJUnitXml(run) {
|
|
21
|
+
const suiteName = xmlEscape(run.suiteId);
|
|
22
|
+
const n = run.goalRuns.length;
|
|
23
|
+
const failures = run.goalRuns.filter((g) => g.verdict === "failed").length;
|
|
24
|
+
const errors = run.goalRuns.filter((g) => g.verdict === "indeterminate").length;
|
|
25
|
+
const time = durationSeconds(run).toFixed(3);
|
|
26
|
+
const cases = [];
|
|
27
|
+
for (const g of run.goalRuns) {
|
|
28
|
+
const name = xmlEscape(g.goalId);
|
|
29
|
+
const className = xmlEscape(`${run.suiteId}.${g.goalId}`);
|
|
30
|
+
const body = `${xmlEscape(g.reasonCode)}: ${xmlEscape(g.summary)}`;
|
|
31
|
+
let inner = "";
|
|
32
|
+
if (g.verdict === "failed") {
|
|
33
|
+
inner = `\n <failure message="${xmlEscape(g.reasonCode)}" type="failed">${body}</failure>`;
|
|
34
|
+
}
|
|
35
|
+
else if (g.verdict === "indeterminate") {
|
|
36
|
+
inner = `\n <error message="${xmlEscape(g.reasonCode)}" type="indeterminate">${body}</error>`;
|
|
37
|
+
}
|
|
38
|
+
if (g.retriesUsed > 0) {
|
|
39
|
+
inner += `\n <!-- retries_used=${g.retriesUsed} -->`;
|
|
40
|
+
}
|
|
41
|
+
if (g.evidenceRefs.length > 0) {
|
|
42
|
+
inner += `\n <!-- evidence: ${xmlEscape(g.evidenceRefs.join(", "))} -->`;
|
|
43
|
+
}
|
|
44
|
+
cases.push(` <testcase name="${name}" classname="${className}" time="0">${inner}\n </testcase>`);
|
|
45
|
+
}
|
|
46
|
+
return `<?xml version="1.0" encoding="UTF-8"?>
|
|
47
|
+
<testsuites>
|
|
48
|
+
<testsuite name="${suiteName}" tests="${n}" failures="${failures}" errors="${errors}" time="${time}">
|
|
49
|
+
${cases.join("\n")}
|
|
50
|
+
</testsuite>
|
|
51
|
+
</testsuites>
|
|
52
|
+
`;
|
|
53
|
+
}
|
|
54
|
+
//# sourceMappingURL=junit.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"junit.js","sourceRoot":"","sources":["../src/junit.ts"],"names":[],"mappings":"AAEA,SAAS,SAAS,CAAC,CAAS;IAC1B,OAAO,CAAC;SACL,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC;SACtB,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC;SACrB,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC;SACrB,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;AAC7B,CAAC;AAED,SAAS,eAAe,CAAC,GAAc;IACrC,IAAI,CAAC,GAAG,CAAC,SAAS,IAAI,CAAC,GAAG,CAAC,OAAO;QAAE,OAAO,CAAC,CAAC;IAC7C,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IACpC,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAClC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,CAAC,CAAC;IAClE,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;AACrC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,aAAa,CAAC,GAAc;IAC1C,MAAM,SAAS,GAAG,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IACzC,MAAM,CAAC,GAAG,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC;IAC9B,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,MAAM,CAAC;IAC3E,MAAM,MAAM,GAAG,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,eAAe,CAAC,CAAC,MAAM,CAAC;IAChF,MAAM,IAAI,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAE7C,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,CAAC,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC;QAC7B,MAAM,IAAI,GAAG,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QACjC,MAAM,SAAS,GAAG,SAAS,CAAC,GAAG,GAAG,CAAC,OAAO,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;QAC1D,MAAM,IAAI,GAAG,GAAG,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;QACnE,IAAI,KAAK,GAAG,EAAE,CAAC;QACf,IAAI,CAAC,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;YAC3B,KAAK,GAAG,6BAA6B,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,mBAAmB,IAAI,YAAY,CAAC;QAClG,CAAC;aAAM,IAAI,CAAC,CAAC,OAAO,KAAK,eAAe,EAAE,CAAC;YACzC,KAAK,GAAG,2BAA2B,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,0BAA0B,IAAI,UAAU,CAAC;QACrG,CAAC;QACD,IAAI,CAAC,CAAC,WAAW,GAAG,CAAC,EAAE,CAAC;YACtB,KAAK,IAAI,6BAA6B,CAAC,CAAC,WAAW,MAAM,CAAC;QAC5D,CAAC;QACD,IAAI,CAAC,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC9B,KAAK,IAAI,0BAA0B,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;QAChF,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,uBAAuB,IAAI,gBAAgB,SAAS,cAAc,KAAK,mBAAmB,CAAC,CAAC;IACzG,CAAC;IAED,OAAO;;qBAEY,SAAS,YAAY,CAAC,eAAe,QAAQ,aAAa,MAAM,WAAW,IAAI;EAClG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC;;;CAGjB,CAAC;AACF,CAAC"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { RunRecord, SuiteReportSlice } from "@restormel/testing-core";
|
|
2
|
+
export interface MarkdownSummaryOptions {
|
|
3
|
+
run: RunRecord;
|
|
4
|
+
suite?: SuiteReportSlice;
|
|
5
|
+
warnings?: string[];
|
|
6
|
+
/** Extra front-matter lines after title (e.g. CI context). */
|
|
7
|
+
preamble?: string[];
|
|
8
|
+
}
|
|
9
|
+
/**
|
|
10
|
+
* Full markdown report for humans (`summary.md`).
|
|
11
|
+
*/
|
|
12
|
+
export declare function buildMarkdownSummary(opts: MarkdownSummaryOptions): string;
|
|
13
|
+
//# sourceMappingURL=markdown-summary.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"markdown-summary.d.ts","sourceRoot":"","sources":["../src/markdown-summary.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAG3E,MAAM,WAAW,sBAAsB;IACrC,GAAG,EAAE,SAAS,CAAC;IACf,KAAK,CAAC,EAAE,gBAAgB,CAAC;IACzB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,8DAA8D;IAC9D,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;CACrB;AAMD;;GAEG;AACH,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,sBAAsB,GAAG,MAAM,CAkHzE"}
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
import { inferFailureBucket } from "./failure-bucket.js";
|
|
2
|
+
function mdEscapeCell(s) {
|
|
3
|
+
return s.replace(/\|/g, "\\|").replace(/\n/g, " ");
|
|
4
|
+
}
|
|
5
|
+
/**
|
|
6
|
+
* Full markdown report for humans (`summary.md`).
|
|
7
|
+
*/
|
|
8
|
+
export function buildMarkdownSummary(opts) {
|
|
9
|
+
const { run, suite, warnings, preamble } = opts;
|
|
10
|
+
const lines = [];
|
|
11
|
+
lines.push(`# Restormel Testing run`);
|
|
12
|
+
lines.push("");
|
|
13
|
+
lines.push(`- **Run id:** \`${run.id}\``);
|
|
14
|
+
lines.push(`- **Verdict:** **${run.verdict}**`);
|
|
15
|
+
lines.push(`- **Suite:** \`${run.suiteId}\``);
|
|
16
|
+
lines.push(`- **Environment:** \`${run.environmentId}\``);
|
|
17
|
+
lines.push(`- **Trigger:** \`${run.trigger}\``);
|
|
18
|
+
if (run.startedAt)
|
|
19
|
+
lines.push(`- **Started:** ${run.startedAt}`);
|
|
20
|
+
if (run.endedAt)
|
|
21
|
+
lines.push(`- **Ended:** ${run.endedAt}`);
|
|
22
|
+
if (run.commitSha)
|
|
23
|
+
lines.push(`- **Commit:** \`${run.commitSha}\``);
|
|
24
|
+
if (run.repository)
|
|
25
|
+
lines.push(`- **Repository:** \`${run.repository}\``);
|
|
26
|
+
lines.push("");
|
|
27
|
+
if (preamble !== undefined && preamble.length > 0) {
|
|
28
|
+
for (const p of preamble) {
|
|
29
|
+
lines.push(p);
|
|
30
|
+
}
|
|
31
|
+
lines.push("");
|
|
32
|
+
}
|
|
33
|
+
if (suite !== undefined) {
|
|
34
|
+
lines.push(`## Suite`);
|
|
35
|
+
lines.push("");
|
|
36
|
+
if (suite.description)
|
|
37
|
+
lines.push(suite.description);
|
|
38
|
+
else
|
|
39
|
+
lines.push(`_(no description)_`);
|
|
40
|
+
lines.push("");
|
|
41
|
+
if (suite.tags !== undefined && suite.tags.length > 0) {
|
|
42
|
+
lines.push(`**Tags:** ${suite.tags.map((t) => `\`${t}\``).join(", ")}`);
|
|
43
|
+
lines.push("");
|
|
44
|
+
}
|
|
45
|
+
lines.push(`**Goals in run:** ${suite.goalCount}`);
|
|
46
|
+
lines.push("");
|
|
47
|
+
}
|
|
48
|
+
const v = run.verdict;
|
|
49
|
+
const counts = { passed: 0, failed: 0, indeterminate: 0 };
|
|
50
|
+
for (const g of run.goalRuns) {
|
|
51
|
+
if (g.verdict === "passed")
|
|
52
|
+
counts.passed++;
|
|
53
|
+
else if (g.verdict === "failed")
|
|
54
|
+
counts.failed++;
|
|
55
|
+
else
|
|
56
|
+
counts.indeterminate++;
|
|
57
|
+
}
|
|
58
|
+
lines.push(`## Verdict summary`);
|
|
59
|
+
lines.push("");
|
|
60
|
+
lines.push(`| Overall | Passed | Failed | Indeterminate | Total |`);
|
|
61
|
+
lines.push(`|--------|--------|--------|---------------|-------|`);
|
|
62
|
+
lines.push(`| **${v}** | ${counts.passed} | ${counts.failed} | ${counts.indeterminate} | ${run.goalRuns.length} |`);
|
|
63
|
+
lines.push("");
|
|
64
|
+
lines.push(`## Goals`);
|
|
65
|
+
lines.push("");
|
|
66
|
+
lines.push(`| Goal | Verdict | Bucket | Retries | Evidence | Reason / summary |`);
|
|
67
|
+
lines.push(`|------|---------|--------|---------|----------|------------------|`);
|
|
68
|
+
for (const g of run.goalRuns) {
|
|
69
|
+
const bucket = inferFailureBucket(g.reasonCode, g.verdict);
|
|
70
|
+
const ev = g.evidenceRefs.length ? g.evidenceRefs.join(", ") : "—";
|
|
71
|
+
const summary = mdEscapeCell(g.summary || g.reasonCode);
|
|
72
|
+
lines.push(`| \`${g.goalId}\` | ${g.verdict} | ${bucket} | ${g.retriesUsed} | ${mdEscapeCell(ev)} | ${summary} |`);
|
|
73
|
+
}
|
|
74
|
+
lines.push("");
|
|
75
|
+
if (run.keysModelMeta !== undefined && run.keysModelMeta.length > 0) {
|
|
76
|
+
lines.push(`## Keys / model metadata (no secrets)`);
|
|
77
|
+
lines.push("");
|
|
78
|
+
lines.push(`| Logical ref | Provider | Model | Invocations | Source |`);
|
|
79
|
+
lines.push(`|-------------|----------|-------|-------------|--------|`);
|
|
80
|
+
for (const m of run.keysModelMeta) {
|
|
81
|
+
lines.push(`| \`${mdEscapeCell(m.logicalRef)}\` | ${m.provider ?? "—"} | ${m.model ?? "—"} | ${m.invocationCount ?? 0} | ${m.resolutionSource ?? "—"} |`);
|
|
82
|
+
}
|
|
83
|
+
lines.push("");
|
|
84
|
+
}
|
|
85
|
+
if (run.judgeInvocationCount !== undefined && run.judgeInvocationCount > 0) {
|
|
86
|
+
lines.push(`## Judge / rubric`);
|
|
87
|
+
lines.push("");
|
|
88
|
+
lines.push(`- **Invocation count:** ${run.judgeInvocationCount} (factual)`);
|
|
89
|
+
lines.push("");
|
|
90
|
+
}
|
|
91
|
+
if (run.costEstimate?.tokenEstimate !== undefined) {
|
|
92
|
+
lines.push(`## Rough token scale (heuristic only)`);
|
|
93
|
+
lines.push("");
|
|
94
|
+
const out = run.costEstimate.tokenEstimate.output ?? 0;
|
|
95
|
+
const inn = run.costEstimate.tokenEstimate.input ?? 0;
|
|
96
|
+
lines.push(`- **Not billing-grade:** input ${inn}, output ${out} — MVP placeholder from invocation count; do not use for spend accounting.`);
|
|
97
|
+
lines.push("");
|
|
98
|
+
}
|
|
99
|
+
if (warnings !== undefined && warnings.length > 0) {
|
|
100
|
+
lines.push(`## Warnings`);
|
|
101
|
+
lines.push("");
|
|
102
|
+
for (const w of warnings) {
|
|
103
|
+
lines.push(`- ${w}`);
|
|
104
|
+
}
|
|
105
|
+
lines.push("");
|
|
106
|
+
}
|
|
107
|
+
lines.push(`## Artefact files`);
|
|
108
|
+
lines.push("");
|
|
109
|
+
lines.push(`In this run directory: \`run.json\`, \`traces.json\`, \`report.json\`, \`summary.md\`, \`github-summary.md\`, \`junit.xml\`, optional \`warnings.txt\`.`);
|
|
110
|
+
lines.push("");
|
|
111
|
+
return lines.join("\n");
|
|
112
|
+
}
|
|
113
|
+
//# sourceMappingURL=markdown-summary.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"markdown-summary.js","sourceRoot":"","sources":["../src/markdown-summary.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AAUzD,SAAS,YAAY,CAAC,CAAS;IAC7B,OAAO,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;AACrD,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,oBAAoB,CAAC,IAA4B;IAC/D,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC;IAChD,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,KAAK,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC;IACtC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,mBAAmB,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC;IAC1C,KAAK,CAAC,IAAI,CAAC,oBAAoB,GAAG,CAAC,OAAO,IAAI,CAAC,CAAC;IAChD,KAAK,CAAC,IAAI,CAAC,kBAAkB,GAAG,CAAC,OAAO,IAAI,CAAC,CAAC;IAC9C,KAAK,CAAC,IAAI,CAAC,wBAAwB,GAAG,CAAC,aAAa,IAAI,CAAC,CAAC;IAC1D,KAAK,CAAC,IAAI,CAAC,oBAAoB,GAAG,CAAC,OAAO,IAAI,CAAC,CAAC;IAChD,IAAI,GAAG,CAAC,SAAS;QAAE,KAAK,CAAC,IAAI,CAAC,kBAAkB,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC;IACjE,IAAI,GAAG,CAAC,OAAO;QAAE,KAAK,CAAC,IAAI,CAAC,gBAAgB,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;IAC3D,IAAI,GAAG,CAAC,SAAS;QAAE,KAAK,CAAC,IAAI,CAAC,mBAAmB,GAAG,CAAC,SAAS,IAAI,CAAC,CAAC;IACpE,IAAI,GAAG,CAAC,UAAU;QAAE,KAAK,CAAC,IAAI,CAAC,uBAAuB,GAAG,CAAC,UAAU,IAAI,CAAC,CAAC;IAC1E,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,IAAI,QAAQ,KAAK,SAAS,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClD,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;YACzB,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAChB,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACxB,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACvB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,IAAI,KAAK,CAAC,WAAW;YAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;;YAChD,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;QACtC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtD,KAAK,CAAC,IAAI,CAAC,aAAa,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACxE,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACjB,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,qBAAqB,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC;QACnD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,MAAM,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC;IACtB,MAAM,MAAM,GAAG,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,CAAC;IAC1D,KAAK,MAAM,CAAC,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC;QAC7B,IAAI,CAAC,CAAC,OAAO,KAAK,QAAQ;YAAE,MAAM,CAAC,MAAM,EAAE,CAAC;aACvC,IAAI,CAAC,CAAC,OAAO,KAAK,QAAQ;YAAE,MAAM,CAAC,MAAM,EAAE,CAAC;;YAC5C,MAAM,CAAC,aAAa,EAAE,CAAC;IAC9B,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;IACjC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,uDAAuD,CAAC,CAAC;IACpE,KAAK,CAAC,IAAI,CAAC,sDAAsD,CAAC,CAAC;IACnE,KAAK,CAAC,IAAI,CACR,OAAO,CAAC,QAAQ,MAAM,CAAC,MAAM,MAAM,MAAM,CAAC,MAAM,MAAM,MAAM,CAAC,aAAa,MAAM,GAAG,CAAC,QAAQ,CAAC,MAAM,IAAI,CACxG,CAAC;IACF,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACvB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,qEAAqE,CAAC,CAAC;IAClF,KAAK,CAAC,IAAI,CAAC,qEAAqE,CAAC,CAAC;IAClF,KAAK,MAAM,CAAC,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC;QAC7B,MAAM,MAAM,GAAG,kBAAkB,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC;QAC3D,MAAM,EAAE,GAAG,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;QACnE,MAAM,OAAO,GAAG,YAAY,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,UAAU,CAAC,CAAC;QACxD,KAAK,CAAC,IAAI,CACR,OAAO,CAAC,CAAC,MAAM,QAAQ,CAAC,CAAC,OAAO,MAAM,MAAM,MAAM,CAAC,CAAC,WAAW,MAAM,YAAY,CAAC,EAAE,CAAC,MAAM,OAAO,IAAI,CACvG,CAAC;IACJ,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,IAAI,GAAG,CAAC,aAAa,KAAK,SAAS,IAAI,GAAG,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACpE,KAAK,CAAC,IAAI,CAAC,uCAAuC,CAAC,CAAC;QACpD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,2DAA2D,CAAC,CAAC;QACxE,KAAK,CAAC,IAAI,CAAC,2DAA2D,CAAC,CAAC;QACxE,KAAK,MAAM,CAAC,IAAI,GAAG,CAAC,aAAa,EAAE,CAAC;YAClC,KAAK,CAAC,IAAI,CACR,OAAO,YAAY,CAAC,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,QAAQ,IAAI,GAAG,MAAM,CAAC,CAAC,KAAK,IAAI,GAAG,MAAM,CAAC,CAAC,eAAe,IAAI,CAAC,MAAM,CAAC,CAAC,gBAAgB,IAAI,GAAG,IAAI,CAC9I,CAAC;QACJ,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,IAAI,GAAG,CAAC,oBAAoB,KAAK,SAAS,IAAI,GAAG,CAAC,oBAAoB,GAAG,CAAC,EAAE,CAAC;QAC3E,KAAK,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;QAChC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,2BAA2B,GAAG,CAAC,oBAAoB,YAAY,CAAC,CAAC;QAC5E,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,IAAI,GAAG,CAAC,YAAY,EAAE,aAAa,KAAK,SAAS,EAAE,CAAC;QAClD,KAAK,CAAC,IAAI,CAAC,uCAAuC,CAAC,CAAC;QACpD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,MAAM,GAAG,GAAG,GAAG,CAAC,YAAY,CAAC,aAAa,CAAC,MAAM,IAAI,CAAC,CAAC;QACvD,MAAM,GAAG,GAAG,GAAG,CAAC,YAAY,CAAC,aAAa,CAAC,KAAK,IAAI,CAAC,CAAC;QACtD,KAAK,CAAC,IAAI,CACR,kCAAkC,GAAG,YAAY,GAAG,4EAA4E,CACjI,CAAC;QACF,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,IAAI,QAAQ,KAAK,SAAS,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClD,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAC1B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;YACzB,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACvB,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;IAChC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,yJAAyJ,CAAC,CAAC;IACtK,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC"}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import type { KeysModelMeta, RunRecord, SuiteReportSlice, Verdict } from "@restormel/testing-core";
|
|
2
|
+
import { type FailureBucket } from "./failure-bucket.js";
|
|
3
|
+
export declare const MVP_REPORT_SCHEMA_VERSION: "1";
|
|
4
|
+
export interface MvpGoalReportRow {
|
|
5
|
+
goal_id: string;
|
|
6
|
+
verdict: Verdict;
|
|
7
|
+
reason_code: string;
|
|
8
|
+
failure_bucket: FailureBucket;
|
|
9
|
+
summary: string;
|
|
10
|
+
retries_used: number;
|
|
11
|
+
evidence_refs: string[];
|
|
12
|
+
}
|
|
13
|
+
export interface MvpJsonReportV1 {
|
|
14
|
+
schema_version: typeof MVP_REPORT_SCHEMA_VERSION;
|
|
15
|
+
generated_at: string;
|
|
16
|
+
run: RunRecord;
|
|
17
|
+
/** Local/CI reproduction hints (no secrets). */
|
|
18
|
+
reproduction?: {
|
|
19
|
+
report_command: string;
|
|
20
|
+
notes?: string;
|
|
21
|
+
};
|
|
22
|
+
suite?: {
|
|
23
|
+
id: string;
|
|
24
|
+
description?: string;
|
|
25
|
+
tags?: string[];
|
|
26
|
+
environment_id: string;
|
|
27
|
+
goal_count: number;
|
|
28
|
+
};
|
|
29
|
+
verdict_summary: {
|
|
30
|
+
overall: Verdict;
|
|
31
|
+
passed: number;
|
|
32
|
+
failed: number;
|
|
33
|
+
indeterminate: number;
|
|
34
|
+
total: number;
|
|
35
|
+
};
|
|
36
|
+
goals: MvpGoalReportRow[];
|
|
37
|
+
/** Relative paths inside the artefact directory. */
|
|
38
|
+
artifact_files: {
|
|
39
|
+
run_json: string;
|
|
40
|
+
traces_json: string;
|
|
41
|
+
report_json: string;
|
|
42
|
+
summary_md: string;
|
|
43
|
+
github_summary_md: string;
|
|
44
|
+
junit_xml: string;
|
|
45
|
+
warnings_txt: string;
|
|
46
|
+
};
|
|
47
|
+
keys_model_meta?: KeysModelMeta[];
|
|
48
|
+
warnings?: string[];
|
|
49
|
+
}
|
|
50
|
+
export declare function buildMvpJsonReport(input: {
|
|
51
|
+
run: RunRecord;
|
|
52
|
+
suite?: SuiteReportSlice;
|
|
53
|
+
warnings?: string[];
|
|
54
|
+
generatedAt?: string;
|
|
55
|
+
reproduction?: MvpJsonReportV1["reproduction"];
|
|
56
|
+
}): MvpJsonReportV1;
|
|
57
|
+
export declare function serializeMvpJsonReport(doc: MvpJsonReportV1): string;
|
|
58
|
+
//# sourceMappingURL=mvp-json-report.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"mvp-json-report.d.ts","sourceRoot":"","sources":["../src/mvp-json-report.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,SAAS,EAAE,gBAAgB,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AACnG,OAAO,EAAsB,KAAK,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAW7E,eAAO,MAAM,yBAAyB,EAAG,GAAY,CAAC;AAEtD,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,OAAO,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,aAAa,CAAC;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,EAAE,CAAC;CACzB;AAED,MAAM,WAAW,eAAe;IAC9B,cAAc,EAAE,OAAO,yBAAyB,CAAC;IACjD,YAAY,EAAE,MAAM,CAAC;IACrB,GAAG,EAAE,SAAS,CAAC;IACf,gDAAgD;IAChD,YAAY,CAAC,EAAE;QACb,cAAc,EAAE,MAAM,CAAC;QACvB,KAAK,CAAC,EAAE,MAAM,CAAC;KAChB,CAAC;IACF,KAAK,CAAC,EAAE;QACN,EAAE,EAAE,MAAM,CAAC;QACX,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;QAChB,cAAc,EAAE,MAAM,CAAC;QACvB,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;IACF,eAAe,EAAE;QACf,OAAO,EAAE,OAAO,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;QACf,aAAa,EAAE,MAAM,CAAC;QACtB,KAAK,EAAE,MAAM,CAAC;KACf,CAAC;IACF,KAAK,EAAE,gBAAgB,EAAE,CAAC;IAC1B,oDAAoD;IACpD,cAAc,EAAE;QACd,QAAQ,EAAE,MAAM,CAAC;QACjB,WAAW,EAAE,MAAM,CAAC;QACpB,WAAW,EAAE,MAAM,CAAC;QACpB,UAAU,EAAE,MAAM,CAAC;QACnB,iBAAiB,EAAE,MAAM,CAAC;QAC1B,SAAS,EAAE,MAAM,CAAC;QAClB,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC;IACF,eAAe,CAAC,EAAE,aAAa,EAAE,CAAC;IAClC,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;CACrB;AAgCD,wBAAgB,kBAAkB,CAAC,KAAK,EAAE;IACxC,GAAG,EAAE,SAAS,CAAC;IACf,KAAK,CAAC,EAAE,gBAAgB,CAAC;IACzB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,eAAe,CAAC,cAAc,CAAC,CAAC;CAChD,GAAG,eAAe,CA4ClB;AAED,wBAAgB,sBAAsB,CAAC,GAAG,EAAE,eAAe,GAAG,MAAM,CAEnE"}
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
import { inferFailureBucket } from "./failure-bucket.js";
|
|
2
|
+
import { GITHUB_SUMMARY_MD, JUNIT_XML, REPORT_JSON, RUN_JSON, SUMMARY_MD, TRACES_JSON, WARNINGS_TXT, } from "./artifact-filenames.js";
|
|
3
|
+
export const MVP_REPORT_SCHEMA_VERSION = "1";
|
|
4
|
+
function verdictCounts(run) {
|
|
5
|
+
let passed = 0;
|
|
6
|
+
let failed = 0;
|
|
7
|
+
let indeterminate = 0;
|
|
8
|
+
for (const g of run.goalRuns) {
|
|
9
|
+
if (g.verdict === "passed")
|
|
10
|
+
passed++;
|
|
11
|
+
else if (g.verdict === "failed")
|
|
12
|
+
failed++;
|
|
13
|
+
else
|
|
14
|
+
indeterminate++;
|
|
15
|
+
}
|
|
16
|
+
return {
|
|
17
|
+
overall: run.verdict,
|
|
18
|
+
passed,
|
|
19
|
+
failed,
|
|
20
|
+
indeterminate,
|
|
21
|
+
total: run.goalRuns.length,
|
|
22
|
+
};
|
|
23
|
+
}
|
|
24
|
+
function goalRows(run) {
|
|
25
|
+
return run.goalRuns.map((g) => ({
|
|
26
|
+
goal_id: g.goalId,
|
|
27
|
+
verdict: g.verdict,
|
|
28
|
+
reason_code: g.reasonCode,
|
|
29
|
+
failure_bucket: inferFailureBucket(g.reasonCode, g.verdict),
|
|
30
|
+
summary: g.summary,
|
|
31
|
+
retries_used: g.retriesUsed,
|
|
32
|
+
evidence_refs: [...g.evidenceRefs],
|
|
33
|
+
}));
|
|
34
|
+
}
|
|
35
|
+
export function buildMvpJsonReport(input) {
|
|
36
|
+
const { run, suite, warnings } = input;
|
|
37
|
+
const generated_at = input.generatedAt ?? new Date().toISOString();
|
|
38
|
+
const doc = {
|
|
39
|
+
schema_version: MVP_REPORT_SCHEMA_VERSION,
|
|
40
|
+
generated_at,
|
|
41
|
+
run,
|
|
42
|
+
verdict_summary: verdictCounts(run),
|
|
43
|
+
goals: goalRows(run),
|
|
44
|
+
artifact_files: {
|
|
45
|
+
run_json: RUN_JSON,
|
|
46
|
+
traces_json: TRACES_JSON,
|
|
47
|
+
report_json: REPORT_JSON,
|
|
48
|
+
summary_md: SUMMARY_MD,
|
|
49
|
+
github_summary_md: GITHUB_SUMMARY_MD,
|
|
50
|
+
junit_xml: JUNIT_XML,
|
|
51
|
+
warnings_txt: WARNINGS_TXT,
|
|
52
|
+
},
|
|
53
|
+
};
|
|
54
|
+
if (suite !== undefined) {
|
|
55
|
+
doc.suite = {
|
|
56
|
+
id: suite.id,
|
|
57
|
+
description: suite.description,
|
|
58
|
+
tags: suite.tags !== undefined && suite.tags.length > 0 ? [...suite.tags] : undefined,
|
|
59
|
+
environment_id: suite.environmentId,
|
|
60
|
+
goal_count: suite.goalCount,
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
if (run.keysModelMeta !== undefined && run.keysModelMeta.length > 0) {
|
|
64
|
+
doc.keys_model_meta = run.keysModelMeta.map((m) => ({ ...m }));
|
|
65
|
+
}
|
|
66
|
+
if (warnings !== undefined && warnings.length > 0) {
|
|
67
|
+
doc.warnings = [...warnings];
|
|
68
|
+
}
|
|
69
|
+
if (input.reproduction !== undefined) {
|
|
70
|
+
doc.reproduction = { ...input.reproduction };
|
|
71
|
+
}
|
|
72
|
+
return doc;
|
|
73
|
+
}
|
|
74
|
+
export function serializeMvpJsonReport(doc) {
|
|
75
|
+
return `${JSON.stringify(doc, null, 2)}\n`;
|
|
76
|
+
}
|
|
77
|
+
//# sourceMappingURL=mvp-json-report.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"mvp-json-report.js","sourceRoot":"","sources":["../src/mvp-json-report.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,kBAAkB,EAAsB,MAAM,qBAAqB,CAAC;AAC7E,OAAO,EACL,iBAAiB,EACjB,SAAS,EACT,WAAW,EACX,QAAQ,EACR,UAAU,EACV,WAAW,EACX,YAAY,GACb,MAAM,yBAAyB,CAAC;AAEjC,MAAM,CAAC,MAAM,yBAAyB,GAAG,GAAY,CAAC;AAkDtD,SAAS,aAAa,CAAC,GAAc;IACnC,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,KAAK,MAAM,CAAC,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC;QAC7B,IAAI,CAAC,CAAC,OAAO,KAAK,QAAQ;YAAE,MAAM,EAAE,CAAC;aAChC,IAAI,CAAC,CAAC,OAAO,KAAK,QAAQ;YAAE,MAAM,EAAE,CAAC;;YACrC,aAAa,EAAE,CAAC;IACvB,CAAC;IACD,OAAO;QACL,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,MAAM;QACN,MAAM;QACN,aAAa;QACb,KAAK,EAAE,GAAG,CAAC,QAAQ,CAAC,MAAM;KAC3B,CAAC;AACJ,CAAC;AAED,SAAS,QAAQ,CAAC,GAAc;IAC9B,OAAO,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAC9B,OAAO,EAAE,CAAC,CAAC,MAAM;QACjB,OAAO,EAAE,CAAC,CAAC,OAAO;QAClB,WAAW,EAAE,CAAC,CAAC,UAAU;QACzB,cAAc,EAAE,kBAAkB,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,OAAO,CAAC;QAC3D,OAAO,EAAE,CAAC,CAAC,OAAO;QAClB,YAAY,EAAE,CAAC,CAAC,WAAW;QAC3B,aAAa,EAAE,CAAC,GAAG,CAAC,CAAC,YAAY,CAAC;KACnC,CAAC,CAAC,CAAC;AACN,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,KAMlC;IACC,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,KAAK,CAAC;IACvC,MAAM,YAAY,GAAG,KAAK,CAAC,WAAW,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAEnE,MAAM,GAAG,GAAoB;QAC3B,cAAc,EAAE,yBAAyB;QACzC,YAAY;QACZ,GAAG;QACH,eAAe,EAAE,aAAa,CAAC,GAAG,CAAC;QACnC,KAAK,EAAE,QAAQ,CAAC,GAAG,CAAC;QACpB,cAAc,EAAE;YACd,QAAQ,EAAE,QAAQ;YAClB,WAAW,EAAE,WAAW;YACxB,WAAW,EAAE,WAAW;YACxB,UAAU,EAAE,UAAU;YACtB,iBAAiB,EAAE,iBAAiB;YACpC,SAAS,EAAE,SAAS;YACpB,YAAY,EAAE,YAAY;SAC3B;KACF,CAAC;IAEF,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACxB,GAAG,CAAC,KAAK,GAAG;YACV,EAAE,EAAE,KAAK,CAAC,EAAE;YACZ,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,IAAI,EAAE,KAAK,CAAC,IAAI,KAAK,SAAS,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS;YACrF,cAAc,EAAE,KAAK,CAAC,aAAa;YACnC,UAAU,EAAE,KAAK,CAAC,SAAS;SAC5B,CAAC;IACJ,CAAC;IAED,IAAI,GAAG,CAAC,aAAa,KAAK,SAAS,IAAI,GAAG,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACpE,GAAG,CAAC,eAAe,GAAG,GAAG,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IACjE,CAAC;IAED,IAAI,QAAQ,KAAK,SAAS,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClD,GAAG,CAAC,QAAQ,GAAG,CAAC,GAAG,QAAQ,CAAC,CAAC;IAC/B,CAAC;IAED,IAAI,KAAK,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;QACrC,GAAG,CAAC,YAAY,GAAG,EAAE,GAAG,KAAK,CAAC,YAAY,EAAE,CAAC;IAC/C,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,GAAoB;IACzD,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC;AAC7C,CAAC"}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import type { RunRecord, TraceEvent } from "@restormel/testing-core";
|
|
2
|
+
export interface LoadedRunArtifacts {
|
|
3
|
+
run: RunRecord;
|
|
4
|
+
traces: TraceEvent[];
|
|
5
|
+
warnings: string[];
|
|
6
|
+
/** Directory containing `run.json` (or the parent when `path` was a file). */
|
|
7
|
+
artifactDir: string;
|
|
8
|
+
}
|
|
9
|
+
/**
|
|
10
|
+
* Load `run.json` (+ optional `traces.json`, `warnings.txt`) from a run directory,
|
|
11
|
+
* or from a direct path to `run.json`.
|
|
12
|
+
*/
|
|
13
|
+
export declare function readRunArtifacts(path: string): Promise<LoadedRunArtifacts | {
|
|
14
|
+
ok: false;
|
|
15
|
+
message: string;
|
|
16
|
+
}>;
|
|
17
|
+
//# sourceMappingURL=read-artifacts.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"read-artifacts.d.ts","sourceRoot":"","sources":["../src/read-artifacts.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAGrE,MAAM,WAAW,kBAAkB;IACjC,GAAG,EAAE,SAAS,CAAC;IACf,MAAM,EAAE,UAAU,EAAE,CAAC;IACrB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,8EAA8E;IAC9E,WAAW,EAAE,MAAM,CAAC;CACrB;AAMD;;;GAGG;AACH,wBAAsB,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,kBAAkB,GAAG;IAAE,EAAE,EAAE,KAAK,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC,CAqDjH"}
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import { readFile, stat } from "node:fs/promises";
|
|
2
|
+
import { dirname, join } from "node:path";
|
|
3
|
+
import { isRunRecord, isTraceEvent } from "@restormel/testing-core";
|
|
4
|
+
import { RUN_JSON, TRACES_JSON, WARNINGS_TXT } from "./write-artifacts.js";
|
|
5
|
+
function isTraceEventArray(value) {
|
|
6
|
+
return Array.isArray(value) && value.every((x) => isTraceEvent(x));
|
|
7
|
+
}
|
|
8
|
+
/**
|
|
9
|
+
* Load `run.json` (+ optional `traces.json`, `warnings.txt`) from a run directory,
|
|
10
|
+
* or from a direct path to `run.json`.
|
|
11
|
+
*/
|
|
12
|
+
export async function readRunArtifacts(path) {
|
|
13
|
+
const st = await stat(path).catch(() => undefined);
|
|
14
|
+
if (st === undefined) {
|
|
15
|
+
return { ok: false, message: `Not found: ${path}` };
|
|
16
|
+
}
|
|
17
|
+
const runJsonPath = st.isDirectory() ? join(path, RUN_JSON) : path;
|
|
18
|
+
const artifactDir = st.isDirectory() ? path : dirname(path);
|
|
19
|
+
let runRaw;
|
|
20
|
+
try {
|
|
21
|
+
runRaw = await readFile(runJsonPath, "utf8");
|
|
22
|
+
}
|
|
23
|
+
catch (e) {
|
|
24
|
+
const msg = e instanceof Error ? e.message : String(e);
|
|
25
|
+
return { ok: false, message: `Cannot read ${runJsonPath}: ${msg}` };
|
|
26
|
+
}
|
|
27
|
+
let parsed;
|
|
28
|
+
try {
|
|
29
|
+
parsed = JSON.parse(runRaw);
|
|
30
|
+
}
|
|
31
|
+
catch {
|
|
32
|
+
return { ok: false, message: `Invalid JSON in ${runJsonPath}` };
|
|
33
|
+
}
|
|
34
|
+
if (!isRunRecord(parsed)) {
|
|
35
|
+
return { ok: false, message: `File is not a valid RunRecord: ${runJsonPath}` };
|
|
36
|
+
}
|
|
37
|
+
const run = parsed;
|
|
38
|
+
let traces = [];
|
|
39
|
+
const tracesPath = join(artifactDir, TRACES_JSON);
|
|
40
|
+
try {
|
|
41
|
+
const tRaw = await readFile(tracesPath, "utf8");
|
|
42
|
+
const tParsed = JSON.parse(tRaw);
|
|
43
|
+
if (isTraceEventArray(tParsed)) {
|
|
44
|
+
traces = tParsed;
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
catch {
|
|
48
|
+
/* traces optional */
|
|
49
|
+
}
|
|
50
|
+
let warnings = [];
|
|
51
|
+
try {
|
|
52
|
+
const wRaw = await readFile(join(artifactDir, WARNINGS_TXT), "utf8");
|
|
53
|
+
warnings = wRaw
|
|
54
|
+
.split("\n")
|
|
55
|
+
.map((l) => l.trimEnd())
|
|
56
|
+
.filter((l) => l.length > 0);
|
|
57
|
+
}
|
|
58
|
+
catch {
|
|
59
|
+
/* optional */
|
|
60
|
+
}
|
|
61
|
+
return { run, traces, warnings, artifactDir };
|
|
62
|
+
}
|
|
63
|
+
//# sourceMappingURL=read-artifacts.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"read-artifacts.js","sourceRoot":"","sources":["../src/read-artifacts.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAEpE,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAU3E,SAAS,iBAAiB,CAAC,KAAc;IACvC,OAAO,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC;AACrE,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,IAAY;IACjD,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IACnD,IAAI,EAAE,KAAK,SAAS,EAAE,CAAC;QACrB,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,cAAc,IAAI,EAAE,EAAE,CAAC;IACtD,CAAC;IAED,MAAM,WAAW,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IACnE,MAAM,WAAW,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAE5D,IAAI,MAAc,CAAC;IACnB,IAAI,CAAC;QACH,MAAM,GAAG,MAAM,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;IAC/C,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,GAAG,GAAG,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QACvD,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,eAAe,WAAW,KAAK,GAAG,EAAE,EAAE,CAAC;IACtE,CAAC;IAED,IAAI,MAAe,CAAC;IACpB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAY,CAAC;IACzC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,mBAAmB,WAAW,EAAE,EAAE,CAAC;IAClE,CAAC;IAED,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC;QACzB,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,kCAAkC,WAAW,EAAE,EAAE,CAAC;IACjF,CAAC;IAED,MAAM,GAAG,GAAG,MAAM,CAAC;IACnB,IAAI,MAAM,GAAiB,EAAE,CAAC;IAC9B,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC;IAClD,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;QAChD,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAY,CAAC;QAC5C,IAAI,iBAAiB,CAAC,OAAO,CAAC,EAAE,CAAC;YAC/B,MAAM,GAAG,OAAO,CAAC;QACnB,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,qBAAqB;IACvB,CAAC;IAED,IAAI,QAAQ,GAAa,EAAE,CAAC;IAC5B,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE,YAAY,CAAC,EAAE,MAAM,CAAC,CAAC;QACrE,QAAQ,GAAG,IAAI;aACZ,KAAK,CAAC,IAAI,CAAC;aACX,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;aACvB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACjC,CAAC;IAAC,MAAM,CAAC;QACP,cAAc;IAChB,CAAC;IAED,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,WAAW,EAAE,CAAC;AAChD,CAAC"}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { RunRecord } from "@restormel/testing-core";
|
|
2
|
+
/**
|
|
3
|
+
* Human-readable summary for terminals and logs. Omits secrets (none in RunRecord).
|
|
4
|
+
*/
|
|
5
|
+
export declare function formatRunSummaryLines(run: RunRecord): string[];
|
|
6
|
+
export declare function formatRunSummary(run: RunRecord): string;
|
|
7
|
+
//# sourceMappingURL=terminal-summary.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"terminal-summary.d.ts","sourceRoot":"","sources":["../src/terminal-summary.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AAGzD;;GAEG;AACH,wBAAgB,qBAAqB,CAAC,GAAG,EAAE,SAAS,GAAG,MAAM,EAAE,CA+B9D;AAED,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,SAAS,GAAG,MAAM,CAEvD"}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import { inferFailureBucket } from "./failure-bucket.js";
|
|
2
|
+
/**
|
|
3
|
+
* Human-readable summary for terminals and logs. Omits secrets (none in RunRecord).
|
|
4
|
+
*/
|
|
5
|
+
export function formatRunSummaryLines(run) {
|
|
6
|
+
const lines = [];
|
|
7
|
+
lines.push(`Run ${run.id}`);
|
|
8
|
+
lines.push(` suite: ${run.suiteId}`);
|
|
9
|
+
lines.push(` environment: ${run.environmentId}`);
|
|
10
|
+
lines.push(` trigger: ${run.trigger}`);
|
|
11
|
+
lines.push(` verdict: ${run.verdict}`);
|
|
12
|
+
if (run.startedAt)
|
|
13
|
+
lines.push(` started: ${run.startedAt}`);
|
|
14
|
+
if (run.endedAt)
|
|
15
|
+
lines.push(` ended: ${run.endedAt}`);
|
|
16
|
+
if (run.commitSha)
|
|
17
|
+
lines.push(` commit: ${run.commitSha}`);
|
|
18
|
+
if (run.repository)
|
|
19
|
+
lines.push(` repository: ${run.repository}`);
|
|
20
|
+
lines.push(" goals:");
|
|
21
|
+
for (const g of run.goalRuns) {
|
|
22
|
+
const ev = g.evidenceRefs.length ? ` [${g.evidenceRefs.length} artefact(s)]` : "";
|
|
23
|
+
const bucket = inferFailureBucket(g.reasonCode, g.verdict);
|
|
24
|
+
lines.push(` - ${g.goalId}: ${g.verdict} (${g.reasonCode}) bucket=${bucket} retries=${g.retriesUsed}${ev}`);
|
|
25
|
+
if (g.summary) {
|
|
26
|
+
lines.push(` ${g.summary}`);
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
if (run.keysModelMeta !== undefined && run.keysModelMeta.length > 0) {
|
|
30
|
+
lines.push(" keys models (no secrets):");
|
|
31
|
+
for (const m of run.keysModelMeta) {
|
|
32
|
+
lines.push(` - ${m.logicalRef} provider=${m.provider ?? "?"} model=${m.model ?? "?"} invocations=${m.invocationCount ?? 0}`);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
return lines;
|
|
36
|
+
}
|
|
37
|
+
export function formatRunSummary(run) {
|
|
38
|
+
return formatRunSummaryLines(run).join("\n");
|
|
39
|
+
}
|
|
40
|
+
//# sourceMappingURL=terminal-summary.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"terminal-summary.js","sourceRoot":"","sources":["../src/terminal-summary.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AAEzD;;GAEG;AACH,MAAM,UAAU,qBAAqB,CAAC,GAAc;IAClD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,CAAC,IAAI,CAAC,OAAO,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC;IAC5B,KAAK,CAAC,IAAI,CAAC,mBAAmB,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;IAC7C,KAAK,CAAC,IAAI,CAAC,mBAAmB,GAAG,CAAC,aAAa,EAAE,CAAC,CAAC;IACnD,KAAK,CAAC,IAAI,CAAC,mBAAmB,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;IAC7C,KAAK,CAAC,IAAI,CAAC,mBAAmB,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;IAC7C,IAAI,GAAG,CAAC,SAAS;QAAE,KAAK,CAAC,IAAI,CAAC,mBAAmB,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC;IAClE,IAAI,GAAG,CAAC,OAAO;QAAE,KAAK,CAAC,IAAI,CAAC,mBAAmB,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;IAC9D,IAAI,GAAG,CAAC,SAAS;QAAE,KAAK,CAAC,IAAI,CAAC,mBAAmB,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC;IAClE,IAAI,GAAG,CAAC,UAAU;QAAE,KAAK,CAAC,IAAI,CAAC,mBAAmB,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;IACpE,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACvB,KAAK,MAAM,CAAC,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC;QAC7B,MAAM,EAAE,GAAG,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,YAAY,CAAC,MAAM,eAAe,CAAC,CAAC,CAAC,EAAE,CAAC;QAClF,MAAM,MAAM,GAAG,kBAAkB,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC;QAC3D,KAAK,CAAC,IAAI,CACR,SAAS,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,UAAU,YAAY,MAAM,YAAY,CAAC,CAAC,WAAW,GAAG,EAAE,EAAE,CACnG,CAAC;QACF,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC;YACd,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;QACnC,CAAC;IACH,CAAC;IACD,IAAI,GAAG,CAAC,aAAa,KAAK,SAAS,IAAI,GAAG,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACpE,KAAK,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;QAC1C,KAAK,MAAM,CAAC,IAAI,GAAG,CAAC,aAAa,EAAE,CAAC;YAClC,KAAK,CAAC,IAAI,CACR,SAAS,CAAC,CAAC,UAAU,aAAa,CAAC,CAAC,QAAQ,IAAI,GAAG,UAAU,CAAC,CAAC,KAAK,IAAI,GAAG,gBAAgB,CAAC,CAAC,eAAe,IAAI,CAAC,EAAE,CACpH,CAAC;QACJ,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,GAAc;IAC7C,OAAO,qBAAqB,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC/C,CAAC"}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import type { RunRecord, TraceEvent } from "@restormel/testing-core";
|
|
2
|
+
import type { SuiteReportSlice } from "@restormel/testing-core";
|
|
3
|
+
export { GITHUB_SUMMARY_MD, JUNIT_XML, REPORT_JSON, RUN_JSON, SUMMARY_MD, TRACES_JSON, WARNINGS_TXT, } from "./artifact-filenames.js";
|
|
4
|
+
export interface RunArtifactPayload {
|
|
5
|
+
run: RunRecord;
|
|
6
|
+
traces: TraceEvent[];
|
|
7
|
+
warnings?: string[];
|
|
8
|
+
}
|
|
9
|
+
export interface RunReportBundleInput extends RunArtifactPayload {
|
|
10
|
+
suite?: SuiteReportSlice;
|
|
11
|
+
reproduction?: {
|
|
12
|
+
report_command: string;
|
|
13
|
+
notes?: string;
|
|
14
|
+
};
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Write raw run record, traces, warnings only (legacy / minimal).
|
|
18
|
+
*/
|
|
19
|
+
export declare function writeRunArtifacts(dir: string, payload: RunArtifactPayload): Promise<void>;
|
|
20
|
+
/**
|
|
21
|
+
* Full MVP artefact set: run.json, traces.json, warnings.txt, report.json, summary.md,
|
|
22
|
+
* github-summary.md (generic), junit.xml.
|
|
23
|
+
*/
|
|
24
|
+
export declare function writeRunReportBundle(dir: string, input: RunReportBundleInput): Promise<void>;
|
|
25
|
+
//# sourceMappingURL=write-artifacts.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"write-artifacts.d.ts","sourceRoot":"","sources":["../src/write-artifacts.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAcrE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAEhE,OAAO,EACL,iBAAiB,EACjB,SAAS,EACT,WAAW,EACX,QAAQ,EACR,UAAU,EACV,WAAW,EACX,YAAY,GACb,MAAM,yBAAyB,CAAC;AAEjC,MAAM,WAAW,kBAAkB;IACjC,GAAG,EAAE,SAAS,CAAC;IACf,MAAM,EAAE,UAAU,EAAE,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;CACrB;AAED,MAAM,WAAW,oBAAqB,SAAQ,kBAAkB;IAC9D,KAAK,CAAC,EAAE,gBAAgB,CAAC;IACzB,YAAY,CAAC,EAAE;QACb,cAAc,EAAE,MAAM,CAAC;QACvB,KAAK,CAAC,EAAE,MAAM,CAAC;KAChB,CAAC;CACH;AAED;;GAEG;AACH,wBAAsB,iBAAiB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,kBAAkB,GAAG,OAAO,CAAC,IAAI,CAAC,CAO/F;AAED;;;GAGG;AACH,wBAAsB,oBAAoB,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,oBAAoB,GAAG,OAAO,CAAC,IAAI,CAAC,CAoClG"}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import { mkdir, writeFile } from "node:fs/promises";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import { GITHUB_SUMMARY_MD, JUNIT_XML, REPORT_JSON, RUN_JSON, SUMMARY_MD, TRACES_JSON, WARNINGS_TXT, } from "./artifact-filenames.js";
|
|
4
|
+
import { buildGithubStepSummaryMarkdown } from "./github-summary.js";
|
|
5
|
+
import { buildJUnitXml } from "./junit.js";
|
|
6
|
+
import { buildMarkdownSummary } from "./markdown-summary.js";
|
|
7
|
+
import { buildMvpJsonReport, serializeMvpJsonReport } from "./mvp-json-report.js";
|
|
8
|
+
export { GITHUB_SUMMARY_MD, JUNIT_XML, REPORT_JSON, RUN_JSON, SUMMARY_MD, TRACES_JSON, WARNINGS_TXT, } from "./artifact-filenames.js";
|
|
9
|
+
/**
|
|
10
|
+
* Write raw run record, traces, warnings only (legacy / minimal).
|
|
11
|
+
*/
|
|
12
|
+
export async function writeRunArtifacts(dir, payload) {
|
|
13
|
+
await mkdir(dir, { recursive: true });
|
|
14
|
+
await writeFile(join(dir, RUN_JSON), `${JSON.stringify(payload.run, null, 2)}\n`, "utf8");
|
|
15
|
+
await writeFile(join(dir, TRACES_JSON), `${JSON.stringify(payload.traces, null, 2)}\n`, "utf8");
|
|
16
|
+
if (payload.warnings !== undefined && payload.warnings.length > 0) {
|
|
17
|
+
await writeFile(join(dir, WARNINGS_TXT), `${payload.warnings.join("\n")}\n`, "utf8");
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Full MVP artefact set: run.json, traces.json, warnings.txt, report.json, summary.md,
|
|
22
|
+
* github-summary.md (generic), junit.xml.
|
|
23
|
+
*/
|
|
24
|
+
export async function writeRunReportBundle(dir, input) {
|
|
25
|
+
await writeRunArtifacts(dir, {
|
|
26
|
+
run: input.run,
|
|
27
|
+
traces: input.traces,
|
|
28
|
+
warnings: input.warnings,
|
|
29
|
+
});
|
|
30
|
+
const mvp = buildMvpJsonReport({
|
|
31
|
+
run: input.run,
|
|
32
|
+
suite: input.suite,
|
|
33
|
+
warnings: input.warnings,
|
|
34
|
+
reproduction: input.reproduction,
|
|
35
|
+
});
|
|
36
|
+
await writeFile(join(dir, REPORT_JSON), serializeMvpJsonReport(mvp), "utf8");
|
|
37
|
+
await writeFile(join(dir, SUMMARY_MD), buildMarkdownSummary({
|
|
38
|
+
run: input.run,
|
|
39
|
+
suite: input.suite,
|
|
40
|
+
warnings: input.warnings,
|
|
41
|
+
}), "utf8");
|
|
42
|
+
await writeFile(join(dir, GITHUB_SUMMARY_MD), buildGithubStepSummaryMarkdown({
|
|
43
|
+
run: input.run,
|
|
44
|
+
suite: input.suite,
|
|
45
|
+
warnings: input.warnings,
|
|
46
|
+
}), "utf8");
|
|
47
|
+
await writeFile(join(dir, JUNIT_XML), buildJUnitXml(input.run), "utf8");
|
|
48
|
+
}
|
|
49
|
+
//# sourceMappingURL=write-artifacts.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"write-artifacts.js","sourceRoot":"","sources":["../src/write-artifacts.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,OAAO,EACL,iBAAiB,EACjB,SAAS,EACT,WAAW,EACX,QAAQ,EACR,UAAU,EACV,WAAW,EACX,YAAY,GACb,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAE,8BAA8B,EAAE,MAAM,qBAAqB,CAAC;AACrE,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC3C,OAAO,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAC7D,OAAO,EAAE,kBAAkB,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AAGlF,OAAO,EACL,iBAAiB,EACjB,SAAS,EACT,WAAW,EACX,QAAQ,EACR,UAAU,EACV,WAAW,EACX,YAAY,GACb,MAAM,yBAAyB,CAAC;AAgBjC;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,GAAW,EAAE,OAA2B;IAC9E,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACtC,MAAM,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,CAAC,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAC1F,MAAM,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,WAAW,CAAC,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAChG,IAAI,OAAO,CAAC,QAAQ,KAAK,SAAS,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClE,MAAM,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,YAAY,CAAC,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IACvF,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,GAAW,EAAE,KAA2B;IACjF,MAAM,iBAAiB,CAAC,GAAG,EAAE;QAC3B,GAAG,EAAE,KAAK,CAAC,GAAG;QACd,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,QAAQ,EAAE,KAAK,CAAC,QAAQ;KACzB,CAAC,CAAC;IAEH,MAAM,GAAG,GAAG,kBAAkB,CAAC;QAC7B,GAAG,EAAE,KAAK,CAAC,GAAG;QACd,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,YAAY,EAAE,KAAK,CAAC,YAAY;KACjC,CAAC,CAAC;IACH,MAAM,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,WAAW,CAAC,EAAE,sBAAsB,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,CAAC;IAE7E,MAAM,SAAS,CACb,IAAI,CAAC,GAAG,EAAE,UAAU,CAAC,EACrB,oBAAoB,CAAC;QACnB,GAAG,EAAE,KAAK,CAAC,GAAG;QACd,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,QAAQ,EAAE,KAAK,CAAC,QAAQ;KACzB,CAAC,EACF,MAAM,CACP,CAAC;IAEF,MAAM,SAAS,CACb,IAAI,CAAC,GAAG,EAAE,iBAAiB,CAAC,EAC5B,8BAA8B,CAAC;QAC7B,GAAG,EAAE,KAAK,CAAC,GAAG;QACd,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,QAAQ,EAAE,KAAK,CAAC,QAAQ;KACzB,CAAC,EACF,MAAM,CACP,CAAC;IAEF,MAAM,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,SAAS,CAAC,EAAE,aAAa,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,CAAC;AAC1E,CAAC"}
|
package/package.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@restormel/testing-report",
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"description": "Run artefacts (JSON) and terminal-friendly summaries for Restormel / Testing.",
|
|
5
|
+
"license": "MIT",
|
|
6
|
+
"repository": {
|
|
7
|
+
"type": "git",
|
|
8
|
+
"url": "https://github.com/Allotment-Technology-Ltd/restormel-testing.git",
|
|
9
|
+
"directory": "packages/report"
|
|
10
|
+
},
|
|
11
|
+
"bugs": {
|
|
12
|
+
"url": "https://github.com/Allotment-Technology-Ltd/restormel-testing/issues"
|
|
13
|
+
},
|
|
14
|
+
"homepage": "https://github.com/Allotment-Technology-Ltd/restormel-testing/tree/main/packages/report#readme",
|
|
15
|
+
"publishConfig": {
|
|
16
|
+
"access": "public"
|
|
17
|
+
},
|
|
18
|
+
"type": "module",
|
|
19
|
+
"main": "./dist/index.js",
|
|
20
|
+
"types": "./dist/index.d.ts",
|
|
21
|
+
"exports": {
|
|
22
|
+
".": {
|
|
23
|
+
"types": "./dist/index.d.ts",
|
|
24
|
+
"import": "./dist/index.js"
|
|
25
|
+
}
|
|
26
|
+
},
|
|
27
|
+
"files": [
|
|
28
|
+
"dist"
|
|
29
|
+
],
|
|
30
|
+
"engines": {
|
|
31
|
+
"node": ">=20"
|
|
32
|
+
},
|
|
33
|
+
"dependencies": {
|
|
34
|
+
"@restormel/testing-core": "0.1.0"
|
|
35
|
+
},
|
|
36
|
+
"devDependencies": {
|
|
37
|
+
"typescript": "^5.7.0"
|
|
38
|
+
},
|
|
39
|
+
"scripts": {
|
|
40
|
+
"build": "tsc -b tsconfig.json",
|
|
41
|
+
"typecheck": "tsc -b tsconfig.json"
|
|
42
|
+
}
|
|
43
|
+
}
|