@tea-agent/loop-agent 0.39.0-next.30 → 0.39.0-next.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +5 -0
- package/dist/build-stamp.json +2 -2
- package/dist/executors/dag-pi-executor.js +7 -4
- package/dist/executors/pi-playwright-cli-tool.js +66 -27
- package/dist/executors/shell-executor.js +25 -8
- package/dist/shared/playwright-cli-command-policy.js +15 -0
- package/dist/task/config-types.js +6 -0
- package/dist/task/source-prepare/parse-intent.js +24 -4
- package/dist/worker/console/chat/pi-console-config.js +81 -10
- package/dist/worker/console/dag-execution-receipt.js +33 -0
- package/dist/worker/console/operator-user-error.js +1 -1
- package/dist/worker/console/static/assets/{abnfDiagram-N423BO3Z-CiGOV5eG.js → abnfDiagram-N423BO3Z-B_Eawmjz.js} +1 -1
- package/dist/worker/console/static/assets/{arc-Ds9Gyfd8.js → arc-CAX7-_9s.js} +1 -1
- package/dist/worker/console/static/assets/{architectureDiagram-T3A2C74G-C6V6uC98.js → architectureDiagram-T3A2C74G-kSHUPU8D.js} +1 -1
- package/dist/worker/console/static/assets/{blockDiagram-VBNYF7ZC-DezF7Ol9.js → blockDiagram-VBNYF7ZC-7uygSzId.js} +1 -1
- package/dist/worker/console/static/assets/{c4Diagram-5PPSVZJV-BCWk3gCT.js → c4Diagram-5PPSVZJV-B7VBFqY8.js} +1 -1
- package/dist/worker/console/static/assets/channel-BqOf3G6O.js +1 -0
- package/dist/worker/console/static/assets/{chunk-2GRJ4B5K-DbYr-Cxy.js → chunk-2GRJ4B5K-CToVTe0G.js} +1 -1
- package/dist/worker/console/static/assets/{chunk-2Q5K7J3B-XSlusW4G.js → chunk-2Q5K7J3B-C3Y6NsZJ.js} +1 -1
- package/dist/worker/console/static/assets/{chunk-5RXB4S5H-BtcKAj9i.js → chunk-5RXB4S5H-D__FOJmB.js} +1 -1
- package/dist/worker/console/static/assets/{chunk-5VM5RSS4-BF57lhXj.js → chunk-5VM5RSS4-DnbdnPQh.js} +1 -1
- package/dist/worker/console/static/assets/{chunk-6Q2QTUOP-DJIlR25o.js → chunk-6Q2QTUOP-fxcnbtmR.js} +1 -1
- package/dist/worker/console/static/assets/{chunk-GF5L2VYU-DklNc7dm.js → chunk-GF5L2VYU-DkWP4DCN.js} +1 -1
- package/dist/worker/console/static/assets/{chunk-JWPE2WC7-fZNtEdQZ.js → chunk-JWPE2WC7-3crbQ-b9.js} +1 -1
- package/dist/worker/console/static/assets/{chunk-KBJHAD2P-BBgLqV4h.js → chunk-KBJHAD2P-AW5fUBHx.js} +1 -1
- package/dist/worker/console/static/assets/{chunk-RYQCIY6F-Dl9e5n_J.js → chunk-RYQCIY6F-BsiQcbN7.js} +1 -1
- package/dist/worker/console/static/assets/{chunk-XXDRQBXY-B0JK51NT.js → chunk-XXDRQBXY-CuRhHBel.js} +1 -1
- package/dist/worker/console/static/assets/classDiagram-JCYQIIEL-BGuxW-g8.js +1 -0
- package/dist/worker/console/static/assets/classDiagram-v2-OCEON4UE-BGuxW-g8.js +1 -0
- package/dist/worker/console/static/assets/{cose-bilkent-JH36ORCC-Ch-ajv4O.js → cose-bilkent-JH36ORCC-VNMSbhR1.js} +1 -1
- package/dist/worker/console/static/assets/{cynefin-VYW2F7L2-xn_NxcfB.js → cynefin-VYW2F7L2-9VXNKhPA.js} +1 -1
- package/dist/worker/console/static/assets/{cynefinDiagram-MW4NZA55-DRD0fyga.js → cynefinDiagram-MW4NZA55-DnA3i2aF.js} +1 -1
- package/dist/worker/console/static/assets/{dagre-VZM6K2ZE-COtnKp4B.js → dagre-VZM6K2ZE-CXf1ON53.js} +1 -1
- package/dist/worker/console/static/assets/{diagram-7IWD3JNH-BQp_CAqG.js → diagram-7IWD3JNH-DkHu5QHM.js} +1 -1
- package/dist/worker/console/static/assets/{diagram-B4RE2ZJO-x0y-vNgq.js → diagram-B4RE2ZJO-b3DVvxzf.js} +1 -1
- package/dist/worker/console/static/assets/{diagram-LBJQPF4R-N2ezwL31.js → diagram-LBJQPF4R-CMtGrmDr.js} +1 -1
- package/dist/worker/console/static/assets/{diagram-Q27KOJAE-CSaxRH6Q.js → diagram-Q27KOJAE-CTUKQdiN.js} +1 -1
- package/dist/worker/console/static/assets/{diagram-UB23O5K3-6Ro7Kpbd.js → diagram-UB23O5K3-XwXry4X0.js} +1 -1
- package/dist/worker/console/static/assets/{ebnfDiagram-BXEA7PRR-CrIIQEAI.js → ebnfDiagram-BXEA7PRR-Dn6cLJ4_.js} +1 -1
- package/dist/worker/console/static/assets/{erDiagram-JOGREHBK-DivKcq8g.js → erDiagram-JOGREHBK-MLibOYKC.js} +1 -1
- package/dist/worker/console/static/assets/{flowDiagram-UKHOOZJN-BXrfDo8y.js → flowDiagram-UKHOOZJN-C3ErZkth.js} +1 -1
- package/dist/worker/console/static/assets/{ganttDiagram-PKOTCBZU-LTTQmMRJ.js → ganttDiagram-PKOTCBZU-CN8OlZId.js} +1 -1
- package/dist/worker/console/static/assets/{gitGraphDiagram-DS77QQ5N-DP3o_5qd.js → gitGraphDiagram-DS77QQ5N-Cq4huIY9.js} +1 -1
- package/dist/worker/console/static/assets/{index-C2jmYBfu.js → index-DSWsdkbR.js} +52 -52
- package/dist/worker/console/static/assets/index-XZm1O8NF.css +1 -0
- package/dist/worker/console/static/assets/{infoDiagram-6WML65LV-wVfSY1WH.js → infoDiagram-6WML65LV-BG_CYrDS.js} +1 -1
- package/dist/worker/console/static/assets/{ishikawaDiagram-WSZJBQD7-BRKfFL0I.js → ishikawaDiagram-WSZJBQD7-DCWDQKJF.js} +1 -1
- package/dist/worker/console/static/assets/{journeyDiagram-NVQOT4AX-DEml3jLb.js → journeyDiagram-NVQOT4AX-DG_VS4XD.js} +1 -1
- package/dist/worker/console/static/assets/{kanban-definition-27J2QSJJ-DTzBmOqN.js → kanban-definition-27J2QSJJ-DkmTDctP.js} +1 -1
- package/dist/worker/console/static/assets/{linear-Cpy1d873.js → linear-C1PpUkLo.js} +1 -1
- package/dist/worker/console/static/assets/{mermaid.core-C1cSavOy.js → mermaid.core-BXExQJbm.js} +5 -5
- package/dist/worker/console/static/assets/{mindmap-definition-FAOFIHXS-BEBSKvYM.js → mindmap-definition-FAOFIHXS-gUlCo9JV.js} +1 -1
- package/dist/worker/console/static/assets/{pegDiagram-VL7TDLO6-BrLs4Oac.js → pegDiagram-VL7TDLO6-JoegdCzT.js} +1 -1
- package/dist/worker/console/static/assets/{pieDiagram-7S7Q4E2Y-B47AZcLa.js → pieDiagram-7S7Q4E2Y-CrJnvJe_.js} +1 -1
- package/dist/worker/console/static/assets/{quadrantDiagram-CIZ2JOQS-s0dVeYR2.js → quadrantDiagram-CIZ2JOQS-BHZqrxkB.js} +1 -1
- package/dist/worker/console/static/assets/{railroadDiagram-AXF67PYL-lW8aJ7Q6.js → railroadDiagram-AXF67PYL-D2uwo4iG.js} +1 -1
- package/dist/worker/console/static/assets/{requirementDiagram-LRYGKXZP-BcHJPf6Q.js → requirementDiagram-LRYGKXZP-Cjg0rXAB.js} +1 -1
- package/dist/worker/console/static/assets/{sankeyDiagram-W5VNT64P-DdegpUUf.js → sankeyDiagram-W5VNT64P-0VbDOPz_.js} +1 -1
- package/dist/worker/console/static/assets/{sequenceDiagram-SI44F4Z6-BxFWqNv8.js → sequenceDiagram-SI44F4Z6-S2UAiI-8.js} +1 -1
- package/dist/worker/console/static/assets/{sizeCapture-X5ZJPWSS-cOdUDPbR.js → sizeCapture-X5ZJPWSS-DQYI8pSg.js} +1 -1
- package/dist/worker/console/static/assets/{stateDiagram-OKZ733FA-DpxlYl8P.js → stateDiagram-OKZ733FA-EIDdI0hz.js} +1 -1
- package/dist/worker/console/static/assets/stateDiagram-v2-UEYNNEHI-BF-RJ1_H.js +1 -0
- package/dist/worker/console/static/assets/{swimlanes-SLNWSIFB-DQOfurE0.js → swimlanes-SLNWSIFB-CfukQalg.js} +2 -2
- package/dist/worker/console/static/assets/swimlanesDiagram-ULZ7WXOC-NswIVCtn.js +8 -0
- package/dist/worker/console/static/assets/{timeline-definition-Z64GVDOM-CIp3tC6V.js → timeline-definition-Z64GVDOM-BOQ3dOWp.js} +1 -1
- package/dist/worker/console/static/assets/{vennDiagram-T6HMQDX7-C-adIPKn.js → vennDiagram-T6HMQDX7-ChhOpX0T.js} +1 -1
- package/dist/worker/console/static/assets/{wardleyDiagram-T6FBY63Y-CiYrar9M.js → wardleyDiagram-T6FBY63Y-BVYvTZDn.js} +1 -1
- package/dist/worker/console/static/assets/{xychartDiagram-ELKLHX3M-WHrdTV0-.js → xychartDiagram-ELKLHX3M-CPRbN4R2.js} +1 -1
- package/dist/worker/console/static/index.html +2 -2
- package/dist/worker/loop-agent/loop-agent-client.js +7 -3
- package/dist/worker/materialize/harness-task-lineage.js +5 -2
- package/dist/worker/observe/static/views/failures.js +5 -2
- package/dist/workflows/dag/dynamic-runtime/loop-until.js +1 -1
- package/dist/workflows/dag/dynamic-runtime/map.js +1 -1
- package/dist/workflows/dag/frontend-repair.js +14 -0
- package/dist/workflows/dag/frontend-risk.js +15 -2
- package/dist/workflows/dag/frontend-test-case-checklist.js +30 -4
- package/dist/workflows/dag/frontend-test-case-manifest.js +11 -4
- package/dist/workflows/dag/frontend-test-case-quality.js +6 -3
- package/dist/workflows/dag/frontend-test-environment-probe.js +10 -7
- package/dist/workflows/dag/frontend-test-html-report.js +3 -1
- package/dist/workflows/dag/frontend-test-l5-report.js +3 -1
- package/dist/workflows/dag/frontend-test-layout.js +159 -0
- package/dist/workflows/dag/frontend-test-result-contract.js +179 -25
- package/dist/workflows/dag/frontend-test-standard-scenarios.js +3 -1
- package/dist/workflows/dag/frontend-verification-trace.js +29 -4
- package/dist/workflows/dag/init-hybrid.js +146 -25
- package/dist/workflows/dag/types.js +12 -0
- package/dist/workflows/dag/validate.js +23 -19
- package/docs/templates/frontend-test-case-checklist.md +2 -2
- package/harness.json +3 -3
- package/package.json +1 -1
- package/skills/fe-test-ui-scout/SKILL.md +65 -0
- package/skills/fe-test-ui-scout/references/ledger-schema.md +62 -0
- package/skills/fe-test-ui-scout/references/recon-protocol.md +54 -0
- package/skills/playwright-cli/SKILL.md +23 -1
- package/skills/playwright-cli-case-generator/SKILL.md +29 -8
- package/dist/worker/console/static/assets/channel-DMvu1nmi.js +0 -1
- package/dist/worker/console/static/assets/classDiagram-JCYQIIEL-DVzM-02m.js +0 -1
- package/dist/worker/console/static/assets/classDiagram-v2-OCEON4UE-DVzM-02m.js +0 -1
- package/dist/worker/console/static/assets/index-C8rOIAJr.css +0 -1
- package/dist/worker/console/static/assets/stateDiagram-v2-UEYNNEHI-BwecGAkM.js +0 -1
- package/dist/worker/console/static/assets/swimlanesDiagram-ULZ7WXOC-D5WW_LbN.js +0 -8
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { readFile, writeFile, mkdir } from "node:fs/promises";
|
|
2
2
|
import path from "node:path";
|
|
3
3
|
import { markdownSection } from "./frontend-test-markdown.js";
|
|
4
|
+
import { defaultFrontendTestLayout, frontendTestCasePath, } from "./frontend-test-layout.js";
|
|
4
5
|
const CASE_ID = /^FE-[A-Za-z0-9]+-\d{3}-(?:core|boundary|flow|backend)$/i;
|
|
5
6
|
const AC_ID = /^AC(?:-[A-Z0-9]+)+$/i;
|
|
6
7
|
const PLACEHOLDER = /\b(?:TODO|TBD|FIXME)\b|结果正确|页面正常|按实际情况处理|验证成功/i;
|
|
@@ -15,7 +16,8 @@ const SECTION_ALIASES = {
|
|
|
15
16
|
function hasList(value) { return /^\s*(?:\d+[.)]|[-*+])\s+\S+/m.test(value); }
|
|
16
17
|
function hasAssertion(value) { return /(?:status|状态|包含|显示|等于|为|可见|不可见|跳转|错误|成功|失败|should|expect|assert|must)/i.test(value); }
|
|
17
18
|
export async function validateFrontendCaseContent(input) {
|
|
18
|
-
const
|
|
19
|
+
const layout = input.layout ?? defaultFrontendTestLayout();
|
|
20
|
+
const root = path.join(input.workspaceRoot, layout.casesDir);
|
|
19
21
|
const manifestPath = path.join(root, "manifest.json");
|
|
20
22
|
const findings = [];
|
|
21
23
|
let manifest;
|
|
@@ -49,7 +51,7 @@ export async function validateFrontendCaseContent(input) {
|
|
|
49
51
|
findings.push({ ruleId: "unknown-ac", caseId: id, detail: `${ac} is not declared by the task` });
|
|
50
52
|
}
|
|
51
53
|
}
|
|
52
|
-
const file = path.join(input.workspaceRoot, item?.casePath ||
|
|
54
|
+
const file = path.join(input.workspaceRoot, item?.casePath || frontendTestCasePath(layout, id));
|
|
53
55
|
let body = "";
|
|
54
56
|
try {
|
|
55
57
|
body = await readFile(file, "utf8");
|
|
@@ -89,7 +91,8 @@ function render(findings) {
|
|
|
89
91
|
}
|
|
90
92
|
export async function writeFrontendCaseQualityReports(input) {
|
|
91
93
|
const result = await validateFrontendCaseContent(input);
|
|
92
|
-
const
|
|
94
|
+
const layout = input.layout ?? defaultFrontendTestLayout();
|
|
95
|
+
const dir = path.join(input.workspaceRoot, input.outputDir ?? layout.reportsDir);
|
|
93
96
|
await mkdir(dir, { recursive: true });
|
|
94
97
|
await writeFile(path.join(dir, "frontend-test-case-quality-advisory.md"), result.markdown, "utf8");
|
|
95
98
|
await writeFile(path.join(dir, "frontend-test-case-quality-advisory.html"), result.html, "utf8");
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { spawnSync } from "node:child_process";
|
|
2
2
|
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
|
3
3
|
import path from "node:path";
|
|
4
|
+
import { defaultFrontendTestLayout, frontendTestContextRel, frontendTestProbeRel, } from "./frontend-test-layout.js";
|
|
4
5
|
export const FRONTEND_TEST_CONTEXT_REL = "testcase/frontend/rag/context.md";
|
|
5
6
|
export const FRONTEND_TEST_PROBE_REL = "testcase/frontend/rag/environment-probe.json";
|
|
6
7
|
const CONNECTION_REFUSED_HINT = "被测服务未在 baseUrl 监听(connection refused)。请先启动本地前端(例如 scripts/serve.sh 或项目约定的 npm start),确认可访问后再从 materialize-frontend-test-execution-shell 续跑。";
|
|
@@ -65,9 +66,9 @@ function resolveBaseUrl(context) {
|
|
|
65
66
|
}
|
|
66
67
|
return null;
|
|
67
68
|
}
|
|
68
|
-
async function writeProbe(workspaceRoot, probe) {
|
|
69
|
-
const probePath = path.join(workspaceRoot,
|
|
70
|
-
const contextPath = path.join(workspaceRoot,
|
|
69
|
+
async function writeProbe(workspaceRoot, probe, layout) {
|
|
70
|
+
const probePath = path.join(workspaceRoot, frontendTestProbeRel(layout));
|
|
71
|
+
const contextPath = path.join(workspaceRoot, frontendTestContextRel(layout));
|
|
71
72
|
await mkdir(path.dirname(probePath), { recursive: true });
|
|
72
73
|
await writeFile(probePath, `${JSON.stringify(probe, null, 2)}\n`, "utf8");
|
|
73
74
|
let context = "";
|
|
@@ -84,14 +85,16 @@ async function writeProbe(workspaceRoot, probe) {
|
|
|
84
85
|
await writeFile(contextPath, next, "utf8");
|
|
85
86
|
}
|
|
86
87
|
export async function probeFrontendTestEnvironment(input) {
|
|
88
|
+
const layout = input.layout ?? defaultFrontendTestLayout();
|
|
87
89
|
const curl = input.curl ?? defaultCurlRunner;
|
|
88
|
-
const
|
|
90
|
+
const contextRel = frontendTestContextRel(layout);
|
|
91
|
+
const contextPath = path.join(input.workspaceRoot, contextRel);
|
|
89
92
|
let context;
|
|
90
93
|
try {
|
|
91
94
|
context = await readFile(contextPath, "utf8");
|
|
92
95
|
}
|
|
93
96
|
catch {
|
|
94
|
-
return { ok: false, stderr: `missing ${
|
|
97
|
+
return { ok: false, stderr: `missing ${contextRel}` };
|
|
95
98
|
}
|
|
96
99
|
const rawUrl = resolveBaseUrl(context);
|
|
97
100
|
if (!rawUrl) {
|
|
@@ -139,7 +142,7 @@ export async function probeFrontendTestEnvironment(input) {
|
|
|
139
142
|
...(extra.hint ? { hint: extra.hint } : {}),
|
|
140
143
|
...(extra.baseUrlSource ? { baseUrlSource: extra.baseUrlSource } : {}),
|
|
141
144
|
};
|
|
142
|
-
await writeProbe(input.workspaceRoot, probe);
|
|
145
|
+
await writeProbe(input.workspaceRoot, probe, layout);
|
|
143
146
|
const payload = {
|
|
144
147
|
blockedReason: reason,
|
|
145
148
|
baseUrl: probe.baseUrlRedacted,
|
|
@@ -218,7 +221,7 @@ export async function probeFrontendTestEnvironment(input) {
|
|
|
218
221
|
curlExit: result.status,
|
|
219
222
|
errorClass: null,
|
|
220
223
|
};
|
|
221
|
-
await writeProbe(input.workspaceRoot, probe);
|
|
224
|
+
await writeProbe(input.workspaceRoot, probe, layout);
|
|
222
225
|
return {
|
|
223
226
|
ok: true,
|
|
224
227
|
stdout: `frontend-test-execution-v1 validated contextBaseUrl=${safe} source=${baseUrlSource} probe=reachable method=${used} httpStatus=${statusNum}`,
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises";
|
|
2
2
|
import path from "node:path";
|
|
3
3
|
import { frontendTestResultContractSchema } from "./frontend-test-result-contract.js";
|
|
4
|
+
import { defaultFrontendTestLayout, } from "./frontend-test-layout.js";
|
|
4
5
|
function escapeHtml(value) {
|
|
5
6
|
return String(value ?? "").replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">").replaceAll('"', """).replaceAll("'", "'");
|
|
6
7
|
}
|
|
@@ -80,7 +81,8 @@ async function writePairAtomic(markdownPath, markdown, htmlPath, html) {
|
|
|
80
81
|
export async function renderFrontendTestHtmlReport(input) {
|
|
81
82
|
const resultPath = path.join(input.runDir, "contracts", "frontend-test-result.json");
|
|
82
83
|
const result = frontendTestResultContractSchema.parse(JSON.parse(await readFile(resultPath, "utf8")));
|
|
83
|
-
const
|
|
84
|
+
const layout = input.layout ?? defaultFrontendTestLayout();
|
|
85
|
+
const outputDir = path.join(input.workspaceRoot, layout.reportsDir);
|
|
84
86
|
const markdownPath = path.join(outputDir, "frontend-test-report.md");
|
|
85
87
|
const htmlPath = path.join(outputDir, "frontend-test-report.html");
|
|
86
88
|
const outcomeLabel = result.outcome === "passed" ? "测试通过" : result.outcome === "failed" ? "测试失败" : "测试未完成";
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises";
|
|
2
2
|
import path from "node:path";
|
|
3
3
|
import { frontendTestResultContractSchema, } from "./frontend-test-result-contract.js";
|
|
4
|
+
import { defaultFrontendTestLayout, } from "./frontend-test-layout.js";
|
|
4
5
|
function ratioMetric(numerator, denominator, threshold, label) {
|
|
5
6
|
if (denominator === 0) {
|
|
6
7
|
return { numerator, denominator, ratio: null, threshold, status: "unavailable", reason: `${label}-denominator-is-zero` };
|
|
@@ -169,7 +170,8 @@ export async function renderFrontendTestL5Report(input) {
|
|
|
169
170
|
const resultPath = path.join(input.runDir, "contracts", "frontend-test-result.json");
|
|
170
171
|
const result = frontendTestResultContractSchema.parse(JSON.parse(await readFile(resultPath, "utf8")));
|
|
171
172
|
const metrics = computeFrontendL5ReportMetrics(result);
|
|
172
|
-
const
|
|
173
|
+
const layout = input.layout ?? defaultFrontendTestLayout();
|
|
174
|
+
const outputDir = path.join(input.workspaceRoot, layout.reportsDir);
|
|
173
175
|
const markdownPath = path.join(outputDir, "frontend-test-l5-dashboard.md");
|
|
174
176
|
const htmlPath = path.join(outputDir, "frontend-test-l5-dashboard.html");
|
|
175
177
|
await writePairAtomic(markdownPath, renderMarkdown(result, metrics), htmlPath, renderHtml(result, metrics));
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
import path from "node:path";
|
|
2
|
+
/**
|
|
3
|
+
* Frontend-test artifact layout (configurable testRoot).
|
|
4
|
+
*
|
|
5
|
+
* Resolved once from `task.frontendTest.testRoot` at DAG compile time and
|
|
6
|
+
* frozen on the spec for the whole run. Writers, gates, Playwright fences
|
|
7
|
+
* and TaskSpec allowedPaths all derive from this single object.
|
|
8
|
+
*
|
|
9
|
+
* Default layout keeps the historical frozen contract:
|
|
10
|
+
* testcase/frontend/rag/**
|
|
11
|
+
* testcase/frontend/cases/**
|
|
12
|
+
* testcase/frontend/evidence/**
|
|
13
|
+
* testcase/frontend/reports/**
|
|
14
|
+
* testcase/frontend/fixtures/**
|
|
15
|
+
*/
|
|
16
|
+
export const FRONTEND_TEST_DEFAULT_TEST_ROOT = "testcase/frontend";
|
|
17
|
+
const FORBIDDEN_TEST_ROOT_PREFIXES = [
|
|
18
|
+
"src",
|
|
19
|
+
"app",
|
|
20
|
+
"lib",
|
|
21
|
+
"apps",
|
|
22
|
+
"packages",
|
|
23
|
+
".harness",
|
|
24
|
+
".git",
|
|
25
|
+
"node_modules",
|
|
26
|
+
"dist",
|
|
27
|
+
"build",
|
|
28
|
+
"docs",
|
|
29
|
+
"website",
|
|
30
|
+
"scripts",
|
|
31
|
+
"test",
|
|
32
|
+
"smoke",
|
|
33
|
+
"knowledge",
|
|
34
|
+
"features",
|
|
35
|
+
"dogfood",
|
|
36
|
+
"examples",
|
|
37
|
+
"bin",
|
|
38
|
+
];
|
|
39
|
+
function normalizeRelPosix(value, label) {
|
|
40
|
+
const trimmed = value.trim();
|
|
41
|
+
if (!trimmed)
|
|
42
|
+
throw new Error(`frontendTest.${label} must not be empty`);
|
|
43
|
+
if (trimmed.includes("\\")) {
|
|
44
|
+
throw new Error(`frontendTest.${label} must use forward slashes: ${value}`);
|
|
45
|
+
}
|
|
46
|
+
if (path.isAbsolute(trimmed) || /^[A-Za-z]:/.test(trimmed)) {
|
|
47
|
+
throw new Error(`frontendTest.${label} must be a repository-relative path: ${value}`);
|
|
48
|
+
}
|
|
49
|
+
const segments = trimmed.split("/").filter((segment) => segment.length > 0);
|
|
50
|
+
if (segments.some((segment) => segment === ".." || segment === ".")) {
|
|
51
|
+
throw new Error(`frontendTest.${label} must not contain "." or ".." segments: ${value}`);
|
|
52
|
+
}
|
|
53
|
+
if (segments.some((segment) => segment.startsWith("."))) {
|
|
54
|
+
throw new Error(`frontendTest.${label} must not target hidden directories: ${value}`);
|
|
55
|
+
}
|
|
56
|
+
if (segments.some((segment) => !/^[A-Za-z0-9][A-Za-z0-9_.-]*$/.test(segment))) {
|
|
57
|
+
throw new Error(`frontendTest.${label} contains unsafe path segments: ${value}`);
|
|
58
|
+
}
|
|
59
|
+
return segments.join("/");
|
|
60
|
+
}
|
|
61
|
+
function assertTestRootAllowed(testRoot) {
|
|
62
|
+
const topSegment = testRoot.split("/")[0];
|
|
63
|
+
if (FORBIDDEN_TEST_ROOT_PREFIXES.some((forbidden) => topSegment === forbidden || topSegment.startsWith(`${forbidden}.`))) {
|
|
64
|
+
throw new Error(`frontendTest.testRoot must not target production/governance trees: ${testRoot}`);
|
|
65
|
+
}
|
|
66
|
+
if (topSegment.startsWith(".")) {
|
|
67
|
+
throw new Error(`frontendTest.testRoot must not target hidden directories: ${testRoot}`);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* Resolve the frozen frontend-test artifact layout from task config.
|
|
72
|
+
* Throws on unsafe or ambiguous configuration (fail-closed).
|
|
73
|
+
*/
|
|
74
|
+
export function resolveFrontendTestLayout(config) {
|
|
75
|
+
const testRoot = config?.testRoot === undefined || config?.testRoot === null
|
|
76
|
+
? FRONTEND_TEST_DEFAULT_TEST_ROOT
|
|
77
|
+
: normalizeRelPosix(config.testRoot, "testRoot");
|
|
78
|
+
assertTestRootAllowed(testRoot);
|
|
79
|
+
const isDefault = testRoot === FRONTEND_TEST_DEFAULT_TEST_ROOT;
|
|
80
|
+
return {
|
|
81
|
+
testRoot,
|
|
82
|
+
ragDir: `${testRoot}/rag`,
|
|
83
|
+
casesDir: `${testRoot}/cases`,
|
|
84
|
+
evidenceDir: `${testRoot}/evidence`,
|
|
85
|
+
reportsDir: `${testRoot}/reports`,
|
|
86
|
+
fixturesDir: `${testRoot}/fixtures`,
|
|
87
|
+
isDefault,
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
/** Historical default layout. Old DAGs without frontendTestLayout use this. */
|
|
91
|
+
export function defaultFrontendTestLayout() {
|
|
92
|
+
return resolveFrontendTestLayout(undefined);
|
|
93
|
+
}
|
|
94
|
+
export function frontendTestLayoutFromSpec(spec) {
|
|
95
|
+
const frozen = spec?.frontendTestLayout;
|
|
96
|
+
if (!frozen)
|
|
97
|
+
return defaultFrontendTestLayout();
|
|
98
|
+
return resolveFrontendTestLayout({ testRoot: frozen.testRoot });
|
|
99
|
+
}
|
|
100
|
+
const DEFAULT_PREFIX = `${FRONTEND_TEST_DEFAULT_TEST_ROOT}/`;
|
|
101
|
+
/**
|
|
102
|
+
* Longest-prefix rewrite of historical `testcase/frontend/` paths.
|
|
103
|
+
* No-op when layout.isDefault so default DAGs stay byte-identical.
|
|
104
|
+
* Never replaces the short token `testcase/` (would collide with backend-test).
|
|
105
|
+
*/
|
|
106
|
+
export function applyFrontendTestLayoutToText(text, layout) {
|
|
107
|
+
if (layout.isDefault)
|
|
108
|
+
return text;
|
|
109
|
+
const replacement = `${layout.testRoot}/`;
|
|
110
|
+
let output = "";
|
|
111
|
+
for (let i = 0; i < text.length;) {
|
|
112
|
+
if (text.startsWith(replacement, i)) {
|
|
113
|
+
output += replacement;
|
|
114
|
+
i += replacement.length;
|
|
115
|
+
continue;
|
|
116
|
+
}
|
|
117
|
+
if (text.startsWith(DEFAULT_PREFIX, i)) {
|
|
118
|
+
output += replacement;
|
|
119
|
+
i += DEFAULT_PREFIX.length;
|
|
120
|
+
continue;
|
|
121
|
+
}
|
|
122
|
+
output += text[i];
|
|
123
|
+
i += 1;
|
|
124
|
+
}
|
|
125
|
+
return output;
|
|
126
|
+
}
|
|
127
|
+
export function frontendTestCasePath(layout, caseId) {
|
|
128
|
+
return `${layout.casesDir}/${caseId}.md`;
|
|
129
|
+
}
|
|
130
|
+
export function frontendTestEvidenceDir(layout, caseId) {
|
|
131
|
+
return `${layout.evidenceDir}/${caseId}/`;
|
|
132
|
+
}
|
|
133
|
+
export function frontendTestEvidencePrefix(layout, caseId) {
|
|
134
|
+
return `${layout.evidenceDir}/${caseId}`;
|
|
135
|
+
}
|
|
136
|
+
export function frontendTestManifestPath(layout) {
|
|
137
|
+
return `${layout.casesDir}/manifest.json`;
|
|
138
|
+
}
|
|
139
|
+
export function frontendTestManifestDraftPath(layout) {
|
|
140
|
+
return `${layout.casesDir}/manifest.draft.json`;
|
|
141
|
+
}
|
|
142
|
+
export function frontendTestContextRel(layout) {
|
|
143
|
+
return `${layout.ragDir}/context.md`;
|
|
144
|
+
}
|
|
145
|
+
export function frontendTestCoverageMapRel(layout) {
|
|
146
|
+
return `${layout.ragDir}/coverage-map.md`;
|
|
147
|
+
}
|
|
148
|
+
export function frontendTestProbeRel(layout) {
|
|
149
|
+
return `${layout.ragDir}/environment-probe.json`;
|
|
150
|
+
}
|
|
151
|
+
export function frontendTestStandardScenariosRel(layout) {
|
|
152
|
+
return `${layout.ragDir}/standard-scenarios.v1.json`;
|
|
153
|
+
}
|
|
154
|
+
export function frontendTestUiAnchorsRel(layout) {
|
|
155
|
+
return `${layout.ragDir}/ui-anchors.md`;
|
|
156
|
+
}
|
|
157
|
+
export function escapeRegexLiteral(value) {
|
|
158
|
+
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
159
|
+
}
|
|
@@ -6,7 +6,27 @@ import { writeDagRunJsonArtifact } from "../../infrastructure/harness/artifact-s
|
|
|
6
6
|
import { readPlaywrightCliReceipts, summarizePlaywrightCliReceipts, } from "../../executors/pi-playwright-cli-tool.js";
|
|
7
7
|
import { validateFrontendCaseContent } from "./frontend-test-case-quality.js";
|
|
8
8
|
import { markdownH1Title, markdownList, markdownSection, } from "./frontend-test-markdown.js";
|
|
9
|
+
import { defaultFrontendTestLayout, frontendTestEvidencePrefix, frontendTestManifestPath, } from "./frontend-test-layout.js";
|
|
9
10
|
export const FRONTEND_TEST_RESULT_SCHEMA_ID = "frontend-test-result-v1";
|
|
11
|
+
/**
|
|
12
|
+
* Truncated acceptance-id shapes observed in generator output: a bare
|
|
13
|
+
* numeric segment ("AC-002" dropped its AC-FE-<FEATURE> prefix) or an
|
|
14
|
+
* AC-FE-<FEATURE> literal without the trailing -NNN sequence number
|
|
15
|
+
* ("AC-FE-CHAT"). Generic short ids like "AC-1"/"AC-FE-001" stay valid;
|
|
16
|
+
* declared-source cross-checks (unknown-ac) remain the primary guard when a
|
|
17
|
+
* sourceBinding exists.
|
|
18
|
+
*/
|
|
19
|
+
export function isTruncatedFrontendAcId(acId) {
|
|
20
|
+
return /^AC-FE-[A-Z]+$/i.test(acId.trim());
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Bare numeric ids ("AC-002") only indicate truncation when no declared AC
|
|
24
|
+
* list exists to adjudicate them: with a sourceBinding they are ordinary
|
|
25
|
+
* unknown-ac findings ("AC-999" may be a legitimately declared id style).
|
|
26
|
+
*/
|
|
27
|
+
export function isTruncatedWhenUndeclaredFrontendAcId(acId) {
|
|
28
|
+
return /^AC-\d{3}$/i.test(acId.trim()) || isTruncatedFrontendAcId(acId);
|
|
29
|
+
}
|
|
10
30
|
const safeRelativePathSchema = z.string().min(1).refine((value) => !path.posix.isAbsolute(value) &&
|
|
11
31
|
!path.win32.isAbsolute(value) &&
|
|
12
32
|
!value.includes("\\") &&
|
|
@@ -57,6 +77,8 @@ export const frontendTestResultContractSchema = z.object({
|
|
|
57
77
|
passed: z.number().int().min(0),
|
|
58
78
|
failed: z.number().int().min(0),
|
|
59
79
|
blocked: z.number().int().min(0),
|
|
80
|
+
/** Stability metric: passed on attempt 0 (no rerun/self-heal retry). */
|
|
81
|
+
firstTryPassed: z.number().int().min(0).optional(),
|
|
60
82
|
}).strict(),
|
|
61
83
|
acceptanceCoverage: z.object({
|
|
62
84
|
covered: z.array(z.string().min(1)),
|
|
@@ -103,14 +125,22 @@ function isSafeEvidenceShellPath(value) {
|
|
|
103
125
|
* backslash interpretation for the former long `node -e` command.
|
|
104
126
|
*/
|
|
105
127
|
export async function validateFrontendCaseEvidence(input) {
|
|
106
|
-
const
|
|
128
|
+
const layout = input.layout ?? defaultFrontendTestLayout();
|
|
129
|
+
const manifestRel = frontendTestManifestPath(layout);
|
|
130
|
+
const manifestPath = path.join(input.workspaceRoot, manifestRel);
|
|
107
131
|
try {
|
|
108
132
|
await stat(manifestPath);
|
|
109
133
|
}
|
|
110
134
|
catch {
|
|
111
|
-
throw new Error(
|
|
135
|
+
throw new Error(`missing ${manifestRel}`);
|
|
136
|
+
}
|
|
137
|
+
let manifest;
|
|
138
|
+
try {
|
|
139
|
+
manifest = JSON.parse(await readFile(manifestPath, "utf8"));
|
|
140
|
+
}
|
|
141
|
+
catch {
|
|
142
|
+
throw new Error("invalid frontend case manifest: malformed JSON");
|
|
112
143
|
}
|
|
113
|
-
const manifest = JSON.parse(await readFile(manifestPath, "utf8"));
|
|
114
144
|
if (!Array.isArray(manifest.cases))
|
|
115
145
|
throw new Error("invalid frontend case manifest");
|
|
116
146
|
const statuses = new Set(["passed", "failed", "blocked"]);
|
|
@@ -120,7 +150,7 @@ export async function validateFrontendCaseEvidence(input) {
|
|
|
120
150
|
const item = rawCase;
|
|
121
151
|
const id = typeof item?.caseId === "string" ? item.caseId : "?";
|
|
122
152
|
const dir = item?.evidenceDir;
|
|
123
|
-
const prefix =
|
|
153
|
+
const prefix = frontendTestEvidencePrefix(layout, id);
|
|
124
154
|
if (!item ||
|
|
125
155
|
typeof item.caseId !== "string" ||
|
|
126
156
|
typeof dir !== "string" ||
|
|
@@ -164,12 +194,15 @@ export async function validateFrontendCaseEvidence(input) {
|
|
|
164
194
|
continue;
|
|
165
195
|
}
|
|
166
196
|
const currentPrefix = `${prefix}/`;
|
|
167
|
-
if (evidencePath.startsWith(
|
|
197
|
+
if ((evidencePath.startsWith(`${layout.evidenceDir}/`) ||
|
|
198
|
+
evidencePath.startsWith("testcase/frontend/evidence/")) &&
|
|
199
|
+
!evidencePath.startsWith(currentPrefix)) {
|
|
168
200
|
hardFail = true;
|
|
169
201
|
issues.push({ ruleId: "cross-case-evidence-path", caseId: id, detail: evidencePath });
|
|
170
202
|
continue;
|
|
171
203
|
}
|
|
172
|
-
const target = evidencePath.startsWith(
|
|
204
|
+
const target = evidencePath.startsWith(`${layout.testRoot}/`) ||
|
|
205
|
+
evidencePath.startsWith("testcase/")
|
|
173
206
|
? path.join(input.workspaceRoot, evidencePath)
|
|
174
207
|
: path.join(input.workspaceRoot, dir, evidencePath);
|
|
175
208
|
if (!(await isNonEmptyFile(target))) {
|
|
@@ -195,11 +228,17 @@ export function evaluatePassedCaseBrowserReceipts(summary, caseId) {
|
|
|
195
228
|
}
|
|
196
229
|
return { ok: true };
|
|
197
230
|
}
|
|
198
|
-
async function
|
|
231
|
+
async function loadCaseBrowserReceiptsWithSummary(input) {
|
|
199
232
|
try {
|
|
200
233
|
const entries = await readdir(input.runDir, { withFileTypes: true });
|
|
201
234
|
// A passed chain must belong to exactly one browser child stream. Never
|
|
202
|
-
// stitch receipts across DAG nodes or accept ambiguous duplicate
|
|
235
|
+
// stitch receipts across DAG nodes or accept ambiguous duplicate
|
|
236
|
+
// authority — with one exception: bounded rerun creates a second,
|
|
237
|
+
// later stream for the same case (the rerun overwrites the disk
|
|
238
|
+
// case-result, so the final attempt is the authoritative one). When
|
|
239
|
+
// multiple streams authorize, prefer the highest rerun round
|
|
240
|
+
// (rerun-frontend-case-rN-<idx> directories); ties within the same
|
|
241
|
+
// round remain ambiguous and fail closed.
|
|
203
242
|
const authorizingStreams = [];
|
|
204
243
|
for (const entry of entries) {
|
|
205
244
|
if (!entry.isDirectory())
|
|
@@ -207,17 +246,99 @@ async function loadCaseBrowserReceiptSummary(input) {
|
|
|
207
246
|
const nodeReceipts = (await readPlaywrightCliReceipts(input.runDir, entry.name))
|
|
208
247
|
.filter((item) => item.caseId === input.caseId);
|
|
209
248
|
const summary = summarizePlaywrightCliReceipts(nodeReceipts);
|
|
210
|
-
if (summary.hasOrderedPassedReceiptChain)
|
|
211
|
-
|
|
249
|
+
if (summary.hasOrderedPassedReceiptChain) {
|
|
250
|
+
const roundMatch = /rerun-frontend-case-r(\d+)-/i.exec(entry.name);
|
|
251
|
+
authorizingStreams.push({
|
|
252
|
+
receipts: nodeReceipts,
|
|
253
|
+
summary,
|
|
254
|
+
round: roundMatch ? Number(roundMatch[1]) : 0,
|
|
255
|
+
});
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
if (authorizingStreams.length === 0) {
|
|
259
|
+
return { receipts: [], summary: summarizePlaywrightCliReceipts([]) };
|
|
212
260
|
}
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
261
|
+
const maxRound = Math.max(...authorizingStreams.map((s) => s.round));
|
|
262
|
+
const finalists = authorizingStreams.filter((s) => s.round === maxRound);
|
|
263
|
+
return finalists.length === 1
|
|
264
|
+
? finalists[0]
|
|
265
|
+
: { receipts: [], summary: summarizePlaywrightCliReceipts([]) };
|
|
266
|
+
}
|
|
267
|
+
catch {
|
|
268
|
+
return { receipts: [], summary: summarizePlaywrightCliReceipts([]) };
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
/**
|
|
272
|
+
* Extract `find "literal" >=N` / `find "literal"` declarations from a case
|
|
273
|
+
* document. Only quoted literals are binding assertions — the count suffix
|
|
274
|
+
* (`>=N`, `==N`) is the expected minimum/exact match count from the ledger.
|
|
275
|
+
* Placeholder literals ("...", "…", "<字面>", "eX") appear in prose discussing
|
|
276
|
+
* the syntax itself; they are documentation, never declarations.
|
|
277
|
+
*/
|
|
278
|
+
export function extractDeclaredFindAssertions(caseMarkdown) {
|
|
279
|
+
const source = stripFencedCodeBlocks(caseMarkdown);
|
|
280
|
+
const out = [];
|
|
281
|
+
const seen = new Set();
|
|
282
|
+
const pattern = /find\s+"((?:[^"\\]|\\.)+)"\s*(>=|==)\s*(\d+)/g;
|
|
283
|
+
const isPlaceholder = (literal) => /^(?:\.{2,}|…+|<[^>]*>|e\d+)$/.test(literal.trim());
|
|
284
|
+
let match;
|
|
285
|
+
while ((match = pattern.exec(source)) !== null) {
|
|
286
|
+
const literal = match[1].replace(/\\(.)/g, "$1");
|
|
287
|
+
if (isPlaceholder(literal))
|
|
288
|
+
continue;
|
|
289
|
+
const operator = match[2];
|
|
290
|
+
const expected = Number(match[3]);
|
|
291
|
+
const key = `${operator}:${literal}`;
|
|
292
|
+
if (seen.has(key))
|
|
293
|
+
continue;
|
|
294
|
+
seen.add(key);
|
|
295
|
+
out.push({ literal, expected, operator });
|
|
296
|
+
}
|
|
297
|
+
return out;
|
|
298
|
+
}
|
|
299
|
+
function stripFencedCodeBlocks(markdown) {
|
|
300
|
+
return markdown.replace(/```[\s\S]*?```/g, "");
|
|
301
|
+
}
|
|
302
|
+
async function readCaseMarkdownForAudit(workspaceRoot, casePath) {
|
|
303
|
+
try {
|
|
304
|
+
return await readFile(path.join(workspaceRoot, casePath), "utf8");
|
|
216
305
|
}
|
|
217
306
|
catch {
|
|
218
|
-
return
|
|
307
|
+
return "";
|
|
219
308
|
}
|
|
220
309
|
}
|
|
310
|
+
/**
|
|
311
|
+
* Controller-side assertion audit: every find-family declaration in the case
|
|
312
|
+
* document must be backed by a successful find receipt whose parsed match
|
|
313
|
+
* count satisfies the declared operator. This moves pass authority from
|
|
314
|
+
* "the model says it asserted" to "receipts prove it asserted".
|
|
315
|
+
*/
|
|
316
|
+
export function auditDeclaredFindAssertions(declared, receipts) {
|
|
317
|
+
if (declared.length === 0)
|
|
318
|
+
return { ok: true };
|
|
319
|
+
const successfulFinds = receipts.filter((receipt) => (receipt.command === "find" || receipt.command === "find-unique") &&
|
|
320
|
+
receipt.exitCode === 0 &&
|
|
321
|
+
!receipt.timedOut);
|
|
322
|
+
const missing = [];
|
|
323
|
+
for (const decl of declared) {
|
|
324
|
+
const backing = successfulFinds.some((receipt) => receipt.argsRedacted.some((arg) => arg === decl.literal || arg === `"${decl.literal}"`) &&
|
|
325
|
+
typeof receipt.matchCount === "number" &&
|
|
326
|
+
(decl.operator === ">="
|
|
327
|
+
? receipt.matchCount >= decl.expected
|
|
328
|
+
: receipt.matchCount === decl.expected));
|
|
329
|
+
if (!backing) {
|
|
330
|
+
missing.push(`${decl.operator}${decl.expected} "${decl.literal.slice(0, 40)}"`);
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
if (missing.length > 0) {
|
|
334
|
+
return {
|
|
335
|
+
ok: false,
|
|
336
|
+
reason: "assertion-not-receipt-backed",
|
|
337
|
+
missing,
|
|
338
|
+
};
|
|
339
|
+
}
|
|
340
|
+
return { ok: true };
|
|
341
|
+
}
|
|
221
342
|
function sha256(content) {
|
|
222
343
|
return createHash("sha256").update(content).digest("hex");
|
|
223
344
|
}
|
|
@@ -228,8 +349,10 @@ function sha256(content) {
|
|
|
228
349
|
* headings. Numbered and bulleted lists are extracted; a missing 测试目的
|
|
229
350
|
* falls back to the case H1 title.
|
|
230
351
|
*/
|
|
231
|
-
async function readFrontendCaseContent(workspaceRoot, casePath) {
|
|
232
|
-
if (!safeRelativePathSchema.safeParse(casePath).success ||
|
|
352
|
+
async function readFrontendCaseContent(workspaceRoot, casePath, layout) {
|
|
353
|
+
if (!safeRelativePathSchema.safeParse(casePath).success ||
|
|
354
|
+
!(casePath.startsWith(`${layout.casesDir}/`) ||
|
|
355
|
+
casePath.startsWith("testcase/frontend/cases/"))) {
|
|
233
356
|
throw new Error(`unsafe frontend case path: ${casePath}`);
|
|
234
357
|
}
|
|
235
358
|
const body = await readFile(path.resolve(workspaceRoot, casePath), "utf8");
|
|
@@ -282,7 +405,7 @@ function assertInside(root, candidate, label) {
|
|
|
282
405
|
throw new Error(`${label} escapes its allowed root`);
|
|
283
406
|
}
|
|
284
407
|
}
|
|
285
|
-
async function readEvidenceFile(repoRoot, evidenceRoot, relative) {
|
|
408
|
+
async function readEvidenceFile(repoRoot, evidenceRoot, relative, layout) {
|
|
286
409
|
if (!safeRelativePathSchema.safeParse(relative).success) {
|
|
287
410
|
throw new Error(`unsafe evidence path: ${relative}`);
|
|
288
411
|
}
|
|
@@ -294,12 +417,16 @@ async function readEvidenceFile(repoRoot, evidenceRoot, relative) {
|
|
|
294
417
|
const normalized = relative;
|
|
295
418
|
const evidenceRootRel = path.relative(repoRootResolved, evidenceRootResolved).replaceAll("\\", "/");
|
|
296
419
|
const isCurrentCaseRepoPath = normalized === evidenceRootRel || normalized.startsWith(`${evidenceRootRel}/`);
|
|
297
|
-
const isRepoRelative = normalized.startsWith(
|
|
420
|
+
const isRepoRelative = (normalized.startsWith(`${layout.testRoot}/`) ||
|
|
421
|
+
normalized.startsWith("testcase/")) &&
|
|
422
|
+
!isCurrentCaseRepoPath;
|
|
298
423
|
const absolute = isRepoRelative
|
|
299
424
|
? path.resolve(repoRootResolved, normalized)
|
|
300
425
|
: path.resolve(evidenceRootResolved, normalized);
|
|
301
426
|
assertInside(repoRootResolved, absolute, "evidence path");
|
|
302
|
-
if (isRepoRelative &&
|
|
427
|
+
if (isRepoRelative &&
|
|
428
|
+
(normalized.startsWith(`${layout.evidenceDir}/`) ||
|
|
429
|
+
normalized.startsWith("testcase/frontend/evidence/"))) {
|
|
303
430
|
throw new Error(`evidence path escapes evidence root: ${relative}`);
|
|
304
431
|
}
|
|
305
432
|
if (!isRepoRelative)
|
|
@@ -322,9 +449,16 @@ export async function materializeFrontendTestResult(input) {
|
|
|
322
449
|
if (!sourceBinding?.taskId || !sourceBinding.sources?.length) {
|
|
323
450
|
throw new Error("frontend-test result requires frozen sourceBinding");
|
|
324
451
|
}
|
|
325
|
-
const
|
|
452
|
+
const layout = input.layout ?? defaultFrontendTestLayout();
|
|
453
|
+
const manifestPath = frontendTestManifestPath(layout);
|
|
326
454
|
const manifestAbsolute = path.resolve(input.workspaceRoot, manifestPath);
|
|
327
|
-
|
|
455
|
+
let manifest;
|
|
456
|
+
try {
|
|
457
|
+
manifest = JSON.parse(await readFile(manifestAbsolute, "utf-8"));
|
|
458
|
+
}
|
|
459
|
+
catch {
|
|
460
|
+
throw new Error("invalid or empty frontend case manifest: malformed JSON");
|
|
461
|
+
}
|
|
328
462
|
if (manifest.schemaVersion !== 1 || !Array.isArray(manifest.cases) || manifest.cases.length === 0) {
|
|
329
463
|
throw new Error("invalid or empty frontend case manifest");
|
|
330
464
|
}
|
|
@@ -332,6 +466,7 @@ export async function materializeFrontendTestResult(input) {
|
|
|
332
466
|
const contentQuality = await validateFrontendCaseContent({
|
|
333
467
|
workspaceRoot: input.workspaceRoot,
|
|
334
468
|
requiredAcIds: sourceBinding.requirementIds ?? [],
|
|
469
|
+
layout,
|
|
335
470
|
});
|
|
336
471
|
advisoryFindings.push(...contentQuality.findings);
|
|
337
472
|
const requiredIds = new Set(sourceBinding.requirementIds ?? []);
|
|
@@ -343,7 +478,7 @@ export async function materializeFrontendTestResult(input) {
|
|
|
343
478
|
advisoryFindings.push({ ruleId: "invalid-manifest-case", detail: "manifest case is missing caseId, acIds, or evidenceDir" });
|
|
344
479
|
continue;
|
|
345
480
|
}
|
|
346
|
-
const evidencePrefix =
|
|
481
|
+
const evidencePrefix = frontendTestEvidencePrefix(layout, item.caseId);
|
|
347
482
|
if (!(item.evidenceDir === evidencePrefix || item.evidenceDir.startsWith(`${evidencePrefix}/`))) {
|
|
348
483
|
throw new Error(`unsafe evidenceDir for ${item.caseId}`);
|
|
349
484
|
}
|
|
@@ -374,7 +509,7 @@ export async function materializeFrontendTestResult(input) {
|
|
|
374
509
|
if (typeof evidencePath !== "string" || evidencePath === "execution.md" && evidence.some((entry) => entry.path.endsWith("/execution.md")))
|
|
375
510
|
continue;
|
|
376
511
|
try {
|
|
377
|
-
evidence.push(await readEvidenceFile(input.workspaceRoot, evidenceRoot, evidencePath));
|
|
512
|
+
evidence.push(await readEvidenceFile(input.workspaceRoot, evidenceRoot, evidencePath, layout));
|
|
378
513
|
}
|
|
379
514
|
catch (error) {
|
|
380
515
|
const message = error instanceof Error ? error.message : String(error);
|
|
@@ -387,7 +522,7 @@ export async function materializeFrontendTestResult(input) {
|
|
|
387
522
|
advisoryFindings.push({ ruleId: "passed-without-browser-evidence", caseId: item.caseId, detail: "passed case has no browser evidence" });
|
|
388
523
|
let blockedReason = status === "blocked" && typeof resultRaw.blockedReason === "string" && resultRaw.blockedReason.trim() ? resultRaw.blockedReason.trim() : undefined;
|
|
389
524
|
if (status === "passed") {
|
|
390
|
-
const receiptSummary = await
|
|
525
|
+
const { receipts: caseReceipts, summary: receiptSummary } = await loadCaseBrowserReceiptsWithSummary({
|
|
391
526
|
runDir: input.runDir,
|
|
392
527
|
caseId: item.caseId,
|
|
393
528
|
});
|
|
@@ -401,8 +536,26 @@ export async function materializeFrontendTestResult(input) {
|
|
|
401
536
|
detail: "passed case lacks controller-owned open + interaction/assertion + cleanup receipts",
|
|
402
537
|
});
|
|
403
538
|
}
|
|
539
|
+
else {
|
|
540
|
+
// Controller-side assertion audit: every `find "literal" >=N` declared
|
|
541
|
+
// in the case document must be backed by a successful find receipt
|
|
542
|
+
// whose parsed matchCount satisfies the operator. Pass authority moves
|
|
543
|
+
// from model prose to receipt evidence.
|
|
544
|
+
const caseMarkdown = await readCaseMarkdownForAudit(input.workspaceRoot, item.casePath);
|
|
545
|
+
const declared = extractDeclaredFindAssertions(caseMarkdown);
|
|
546
|
+
const audit = auditDeclaredFindAssertions(declared, caseReceipts);
|
|
547
|
+
if (!audit.ok) {
|
|
548
|
+
status = "blocked";
|
|
549
|
+
blockedReason = audit.reason;
|
|
550
|
+
advisoryFindings.push({
|
|
551
|
+
ruleId: "assertion-not-receipt-backed",
|
|
552
|
+
caseId: item.caseId,
|
|
553
|
+
detail: `declared find assertions without receipt backing: ${audit.missing.join("; ")}`,
|
|
554
|
+
});
|
|
555
|
+
}
|
|
556
|
+
}
|
|
404
557
|
}
|
|
405
|
-
const { caseContent, testPoints } = await readFrontendCaseContent(input.workspaceRoot, item.casePath);
|
|
558
|
+
const { caseContent, testPoints } = await readFrontendCaseContent(input.workspaceRoot, item.casePath, layout);
|
|
406
559
|
// Parse bounded fixed sections from execution.md. Missing optional
|
|
407
560
|
// summary/steps are advisory only and never downgrade a passed status.
|
|
408
561
|
let executionSummary = "";
|
|
@@ -451,6 +604,7 @@ export async function materializeFrontendTestResult(input) {
|
|
|
451
604
|
passed: cases.filter((item) => item.status === "passed").length,
|
|
452
605
|
failed: cases.filter((item) => item.status === "failed").length,
|
|
453
606
|
blocked: cases.filter((item) => item.status === "blocked").length,
|
|
607
|
+
firstTryPassed: cases.filter((item) => item.status === "passed" && !item.rerunAttempt).length,
|
|
454
608
|
};
|
|
455
609
|
const outcome = totals.failed > 0 ? "failed" : totals.blocked > 0 || missing.length > 0 ? "incomplete" : "passed";
|
|
456
610
|
const result = frontendTestResultContractSchema.parse({
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { copyFile, mkdir, readFile, writeFile } from "node:fs/promises";
|
|
2
2
|
import path from "node:path";
|
|
3
|
+
import { defaultFrontendTestLayout, frontendTestStandardScenariosRel, } from "./frontend-test-layout.js";
|
|
3
4
|
export const FRONTEND_TEST_STANDARD_SCENARIOS_FILENAME = "frontend-test-standard-scenarios.v1.json";
|
|
4
5
|
export const FRONTEND_TEST_STANDARD_SCENARIOS_DEST = "testcase/frontend/rag/standard-scenarios.v1.json";
|
|
5
6
|
const MINIMAL_STANDARD_SCENARIOS = {
|
|
@@ -47,7 +48,8 @@ async function readGovernanceRoot(workspaceRoot) {
|
|
|
47
48
|
}
|
|
48
49
|
}
|
|
49
50
|
export async function copyFrontendTestStandardScenarios(input) {
|
|
50
|
-
const
|
|
51
|
+
const layout = input.layout ?? defaultFrontendTestLayout();
|
|
52
|
+
const destRel = frontendTestStandardScenariosRel(layout);
|
|
51
53
|
const destAbs = path.join(input.workspaceRoot, destRel);
|
|
52
54
|
const candidates = listFrontendTestStandardScenarioCandidates({
|
|
53
55
|
governanceRoot: await readGovernanceRoot(input.workspaceRoot),
|