@tea-agent/loop-agent 0.25.4 → 0.25.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +6 -0
- package/CHANGELOG.md +60 -0
- package/dist/commands/client-recovery.js +209 -62
- package/dist/commands/init.js +68 -129
- package/dist/executors/dag-pi-executor.js +80 -15
- package/dist/executors/model-routing.js +1 -1
- package/dist/executors/shell-executor.js +127 -0
- package/dist/executors/shell-write-guard.js +21 -7
- package/dist/worker/console/repo-fingerprint.js +7 -1
- package/dist/workflows/dag/backend-test-case-coverage-analysis.js +1281 -0
- package/dist/workflows/dag/backend-test-case-manifest.js +59 -1
- package/dist/workflows/dag/backend-test-markdown-workflow.js +236 -16
- package/dist/workflows/dag/convergence/controller.js +134 -9
- package/dist/workflows/dag/frontend-test-l5-report.js +138 -0
- package/dist/workflows/dag/init-hybrid.js +270 -80
- package/dist/workflows/dag/node-execution.js +64 -11
- package/dist/workflows/dag/prompt.js +118 -4
- package/dist/workflows/dag/retry-policy.js +5 -4
- package/dist/workflows/dag/scheduler.js +32 -5
- package/dist/workflows/dag/types.js +7 -4
- package/dist/workflows/dag/validate.js +3 -2
- package/docs/architecture/dag-execution.md +7 -4
- package/docs/architecture/runtime-boundaries.md +1 -1
- package/docs/init-surface.manifest.json +3 -1
- package/docs/templates/README.md +1 -0
- package/docs/templates/agent-dag.base.json +1 -1
- package/docs/templates/agent-dag.final-verification.json +1 -1
- package/docs/templates/agent-dag.supervised-implementation.json +1 -1
- package/docs/templates/backend-test-dag.json +41 -14
- package/docs/templates/frontend-test-dag.json +32 -2
- package/docs/templates/hybrid-dag.json +1 -1
- package/docs/templates/init-managed-agents.md +137 -0
- package/examples/decision-gate-agent-dag.json +1 -1
- package/examples/example-dag.json +1 -1
- package/examples/hybrid-loop-agent-dag.json +1 -1
- package/harness.json +1 -1
- package/package.json +1 -1
- package/skills/loop-agent/references/command-reference.md +5 -4
- package/skills/loop-agent/references/hybrid-dag.md +2 -2
- package/skills/loop-agent/references/model-routing.md +1 -1
|
@@ -0,0 +1,1281 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { access, mkdir, readFile, readdir, writeFile } from "node:fs/promises";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { z } from "zod";
|
|
5
|
+
import YAML from "yaml";
|
|
6
|
+
import { backendTestCaseManifestSchema, computeCaseManifestCoverageSummary, } from "./backend-test-case-manifest.js";
|
|
7
|
+
const CASE_HEADING = /^##\s+(BE-[A-Z0-9_-]+-\d{3})\b.*$/gm;
|
|
8
|
+
const CASE_ID_IN_TEXT = /\bBE-[A-Z0-9_-]+-\d{3}\b/g;
|
|
9
|
+
const NON_CANONICAL_CASE_HEADING = /^##\s+(BE-[A-Z0-9_-]+-(?:\d{2}|\d{2,3}[A-Z]+))\b.*$/gm;
|
|
10
|
+
const AC_ID_IN_TEXT = /\bAC-[A-Z0-9]+(?:-[A-Z0-9]+)*\b/g;
|
|
11
|
+
const RULE_KEY = /^(?:AC|REQ|BR|API|GET|POST|PUT|PATCH|DELETE)-[A-Z0-9]+(?:-[A-Z0-9]+)*$/;
|
|
12
|
+
const TEST_POINT = /^TP-[A-Z0-9]+(?:-[A-Z0-9]+)*$/;
|
|
13
|
+
const COVERAGE_HEADERS = [
|
|
14
|
+
"Rule Key",
|
|
15
|
+
"Priority",
|
|
16
|
+
"Source",
|
|
17
|
+
"Endpoint/Field",
|
|
18
|
+
"Dimension",
|
|
19
|
+
"Rule",
|
|
20
|
+
"Required Test Points",
|
|
21
|
+
"Case IDs",
|
|
22
|
+
"Status",
|
|
23
|
+
];
|
|
24
|
+
const CASE_SECTION_ALIASES = {
|
|
25
|
+
acceptance: ["验收标准", "Acceptance Criteria"],
|
|
26
|
+
rules: ["覆盖规则", "Coverage Rules"],
|
|
27
|
+
testPoints: ["测试点", "Test Points"],
|
|
28
|
+
scenarioTypes: ["场景类型", "Scenario Types", "Scenario Type"],
|
|
29
|
+
automation: ["自动化映射", "自动化说明", "Automation Notes"],
|
|
30
|
+
};
|
|
31
|
+
const TEST_POINT_BINDING_MODES = ["variant", "assertion", "cross-cutting"];
|
|
32
|
+
const CHANGE_CLASSIFICATIONS = [
|
|
33
|
+
"new-operation",
|
|
34
|
+
"contract-change",
|
|
35
|
+
"behavior-change",
|
|
36
|
+
"bugfix",
|
|
37
|
+
"implementation-optimization",
|
|
38
|
+
];
|
|
39
|
+
const COVERAGE_POLICIES = [
|
|
40
|
+
"full-contract",
|
|
41
|
+
"affected-contract-full",
|
|
42
|
+
"affected-behavior-full",
|
|
43
|
+
"reproduction-plus-neighbors",
|
|
44
|
+
"change-focused-plus-regression-floor",
|
|
45
|
+
];
|
|
46
|
+
const COVERAGE_POLICY_BY_CLASSIFICATION = {
|
|
47
|
+
"new-operation": "full-contract",
|
|
48
|
+
"contract-change": "affected-contract-full",
|
|
49
|
+
"behavior-change": "affected-behavior-full",
|
|
50
|
+
bugfix: "reproduction-plus-neighbors",
|
|
51
|
+
"implementation-optimization": "change-focused-plus-regression-floor",
|
|
52
|
+
};
|
|
53
|
+
const REQUIRED_REGRESSION_FLOOR = {
|
|
54
|
+
"new-operation": [],
|
|
55
|
+
"contract-change": ["main-success-path", "unchanged-response-shape", "changed-contract-boundaries"],
|
|
56
|
+
"behavior-change": ["main-success-path", "unchanged-response-shape", "affected-state-transition"],
|
|
57
|
+
bugfix: ["main-success-path", "unchanged-response-shape", "defect-reproduction", "adjacent-boundary"],
|
|
58
|
+
"implementation-optimization": ["main-success-path", "unchanged-response-shape"],
|
|
59
|
+
};
|
|
60
|
+
const AUTOMATION_BINDING_LABELS = {
|
|
61
|
+
variant: ["变体测试点", "Variant Test Points"],
|
|
62
|
+
assertion: ["场景断言测试点", "Assertion Test Points"],
|
|
63
|
+
"cross-cutting": ["横切证据测试点", "Cross-Cutting Test Points"],
|
|
64
|
+
};
|
|
65
|
+
const sha256Schema = z.string().regex(/^[a-f0-9]{64}$/);
|
|
66
|
+
const inputFileSchema = z.object({ path: z.string().min(1), sha256: sha256Schema }).strict();
|
|
67
|
+
const sourceBindingSchema = z.object({
|
|
68
|
+
taskId: z.string().min(1),
|
|
69
|
+
requirementPath: z.string().min(1),
|
|
70
|
+
requirementSha256: sha256Schema,
|
|
71
|
+
referencePaths: z.array(z.string().min(1)),
|
|
72
|
+
requirementIds: z.array(z.string().min(1)),
|
|
73
|
+
}).strict();
|
|
74
|
+
const coverageCaseSchema = z.object({
|
|
75
|
+
caseId: z.string().min(1),
|
|
76
|
+
title: z.string().min(1),
|
|
77
|
+
markdownPath: z.string().min(1),
|
|
78
|
+
acIds: z.array(z.string()),
|
|
79
|
+
ruleKeys: z.array(z.string()),
|
|
80
|
+
testPoints: z.array(z.string()),
|
|
81
|
+
testPointBindings: z.array(z.object({
|
|
82
|
+
testPoint: z.string(),
|
|
83
|
+
mode: z.enum(TEST_POINT_BINDING_MODES),
|
|
84
|
+
}).strict()),
|
|
85
|
+
unclassifiedTestPoints: z.array(z.string()),
|
|
86
|
+
duplicateBindingTestPoints: z.array(z.string()),
|
|
87
|
+
scenarioTypes: z.array(z.string()),
|
|
88
|
+
declaredScripts: z.array(z.string()),
|
|
89
|
+
declaredPrimarySymbols: z.array(z.string()),
|
|
90
|
+
}).strict();
|
|
91
|
+
const coverageRuleSchema = z.object({
|
|
92
|
+
ruleKey: z.string().min(1),
|
|
93
|
+
priority: z.enum(["P0", "P1", "P2"]),
|
|
94
|
+
source: z.string().min(1),
|
|
95
|
+
endpointField: z.string().min(1),
|
|
96
|
+
dimension: z.string().min(1),
|
|
97
|
+
rule: z.string().min(1),
|
|
98
|
+
requiredTestPoints: z.array(z.string()),
|
|
99
|
+
caseIds: z.array(z.string()),
|
|
100
|
+
declaredStatus: z.enum(["COVERED", "PARTIAL", "GAP", "CONFLICT"]),
|
|
101
|
+
coveredTestPoints: z.array(z.string()),
|
|
102
|
+
missingTestPoints: z.array(z.string()),
|
|
103
|
+
status: z.enum(["COVERED", "PARTIAL", "GAP", "CONFLICT"]),
|
|
104
|
+
}).strict();
|
|
105
|
+
const coverageScopeSchema = z.object({
|
|
106
|
+
changeClassification: z.enum(CHANGE_CLASSIFICATIONS),
|
|
107
|
+
coveragePolicy: z.enum(COVERAGE_POLICIES),
|
|
108
|
+
affectedOperations: z.array(z.string().min(1)),
|
|
109
|
+
affectedRuleKeys: z.array(z.string().min(1)),
|
|
110
|
+
regressionFloor: z.array(z.string().min(1)),
|
|
111
|
+
scopeEvidence: z.array(z.string().min(1)),
|
|
112
|
+
completenessClaim: z.enum(["affected-operations-full", "affected-scope"]),
|
|
113
|
+
}).strict();
|
|
114
|
+
export const backendTestCaseCoverageFactsSchema = z.object({
|
|
115
|
+
schemaId: z.literal("backend-test-case-coverage-facts-v3"),
|
|
116
|
+
schemaVersion: z.literal(3),
|
|
117
|
+
taskId: z.string().min(1),
|
|
118
|
+
status: z.enum(["PASS", "FAIL", "UNAVAILABLE"]),
|
|
119
|
+
sourceBinding: sourceBindingSchema,
|
|
120
|
+
inputFiles: z.array(inputFileSchema),
|
|
121
|
+
coverageScope: coverageScopeSchema,
|
|
122
|
+
summary: z.object({
|
|
123
|
+
explicitAcCount: z.number().int().min(0),
|
|
124
|
+
coveredAcCount: z.number().int().min(0),
|
|
125
|
+
ruleCount: z.number().int().min(0),
|
|
126
|
+
coveredRuleCount: z.number().int().min(0),
|
|
127
|
+
testPointCount: z.number().int().min(0),
|
|
128
|
+
coveredTestPointCount: z.number().int().min(0),
|
|
129
|
+
enumValueCount: z.number().int().min(0),
|
|
130
|
+
coveredEnumValueCount: z.number().int().min(0),
|
|
131
|
+
invalidEquivalenceClassCount: z.number().int().min(0),
|
|
132
|
+
coveredInvalidEquivalenceClassCount: z.number().int().min(0),
|
|
133
|
+
boundaryPointCount: z.number().int().min(0),
|
|
134
|
+
coveredBoundaryPointCount: z.number().int().min(0),
|
|
135
|
+
formatClassCount: z.number().int().min(0),
|
|
136
|
+
coveredFormatClassCount: z.number().int().min(0),
|
|
137
|
+
businessStateCount: z.number().int().min(0),
|
|
138
|
+
coveredBusinessStateCount: z.number().int().min(0),
|
|
139
|
+
gapCount: z.number().int().min(0),
|
|
140
|
+
conflictCount: z.number().int().min(0),
|
|
141
|
+
variantTestPointCount: z.number().int().min(0),
|
|
142
|
+
assertionTestPointCount: z.number().int().min(0),
|
|
143
|
+
crossCuttingTestPointCount: z.number().int().min(0),
|
|
144
|
+
unclassifiedTestPointCount: z.number().int().min(0),
|
|
145
|
+
duplicateBindingTestPointCount: z.number().int().min(0),
|
|
146
|
+
}).strict(),
|
|
147
|
+
cases: z.array(coverageCaseSchema),
|
|
148
|
+
rules: z.array(coverageRuleSchema),
|
|
149
|
+
findings: z.array(z.string()),
|
|
150
|
+
evidenceGaps: z.array(z.string()),
|
|
151
|
+
conflicts: z.array(z.string()),
|
|
152
|
+
}).strict();
|
|
153
|
+
const correspondenceEntrySchema = z.object({
|
|
154
|
+
markdownModule: z.string().min(1),
|
|
155
|
+
caseId: z.string().optional(),
|
|
156
|
+
testPoints: z.array(z.string()),
|
|
157
|
+
variantTestPoints: z.array(z.string()),
|
|
158
|
+
assertionTestPoints: z.array(z.string()),
|
|
159
|
+
crossCuttingTestPoints: z.array(z.string()),
|
|
160
|
+
mappedTestPoints: z.array(z.string()),
|
|
161
|
+
declaredScript: z.string().optional(),
|
|
162
|
+
declaredPrimarySymbol: z.string().optional(),
|
|
163
|
+
expectedScript: z.string().min(1),
|
|
164
|
+
actualScripts: z.array(z.string()),
|
|
165
|
+
pytestSymbols: z.array(z.string()),
|
|
166
|
+
parameterIds: z.array(z.string()),
|
|
167
|
+
cardinality: z.enum(["1:1", "1:0", "1:N", "0:1"]),
|
|
168
|
+
status: z.enum([
|
|
169
|
+
"EXACT_1_TO_1",
|
|
170
|
+
"MISSING_PYTEST",
|
|
171
|
+
"MULTIPLE_PYTEST",
|
|
172
|
+
"EXTRA_PYTEST",
|
|
173
|
+
"SCRIPT_MISMATCH",
|
|
174
|
+
"SYMBOL_MISSING_CASE_ID",
|
|
175
|
+
"VARIANT_PARAMETER_MISSING",
|
|
176
|
+
"ASSERTION_BINDING_MISSING",
|
|
177
|
+
"CROSS_CUTTING_EVIDENCE_MISSING",
|
|
178
|
+
"TEST_POINT_BINDING_DUPLICATE",
|
|
179
|
+
"TEST_POINT_BINDING_EXTRA",
|
|
180
|
+
]),
|
|
181
|
+
findings: z.array(z.string()),
|
|
182
|
+
}).strict();
|
|
183
|
+
export const backendTestMarkdownPytestCorrespondenceFactsSchema = z.object({
|
|
184
|
+
schemaId: z.literal("backend-test-markdown-pytest-correspondence-facts-v2"),
|
|
185
|
+
schemaVersion: z.literal(2),
|
|
186
|
+
taskId: z.string().min(1),
|
|
187
|
+
status: z.enum(["PASS", "FAIL", "UNAVAILABLE"]),
|
|
188
|
+
inputFiles: z.array(inputFileSchema),
|
|
189
|
+
summary: z.object({
|
|
190
|
+
markdownModuleCount: z.number().int().min(0),
|
|
191
|
+
exactModuleCount: z.number().int().min(0),
|
|
192
|
+
markdownCaseCount: z.number().int().min(0),
|
|
193
|
+
exactCorrespondenceCount: z.number().int().min(0),
|
|
194
|
+
primarySymbolCount: z.number().int().min(0),
|
|
195
|
+
missingPytestCount: z.number().int().min(0),
|
|
196
|
+
multiplePytestCount: z.number().int().min(0),
|
|
197
|
+
extraPytestCount: z.number().int().min(0),
|
|
198
|
+
scriptMismatchCount: z.number().int().min(0),
|
|
199
|
+
testPointCount: z.number().int().min(0),
|
|
200
|
+
mappedTestPointCount: z.number().int().min(0),
|
|
201
|
+
variantTestPointCount: z.number().int().min(0),
|
|
202
|
+
assertionTestPointCount: z.number().int().min(0),
|
|
203
|
+
crossCuttingTestPointCount: z.number().int().min(0),
|
|
204
|
+
unclassifiedTestPointCount: z.number().int().min(0),
|
|
205
|
+
duplicateBindingTestPointCount: z.number().int().min(0),
|
|
206
|
+
}).strict(),
|
|
207
|
+
entries: z.array(correspondenceEntrySchema),
|
|
208
|
+
findings: z.array(z.string()),
|
|
209
|
+
}).strict();
|
|
210
|
+
function isRecord(value) {
|
|
211
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
212
|
+
}
|
|
213
|
+
function openApiToken(value) {
|
|
214
|
+
return value.replace(/[{}]/g, "").replace(/[^A-Za-z0-9]+/g, "-").replace(/^-+|-+$/g, "").toUpperCase();
|
|
215
|
+
}
|
|
216
|
+
function resolveOpenApiSchema(document, value) {
|
|
217
|
+
if (!isRecord(value))
|
|
218
|
+
return undefined;
|
|
219
|
+
if (typeof value.$ref !== "string" || !value.$ref.startsWith("#/"))
|
|
220
|
+
return value;
|
|
221
|
+
let current = document;
|
|
222
|
+
for (const segment of value.$ref.slice(2).split("/")) {
|
|
223
|
+
if (!isRecord(current))
|
|
224
|
+
return undefined;
|
|
225
|
+
current = current[segment.replace(/~1/g, "/").replace(/~0/g, "~")];
|
|
226
|
+
}
|
|
227
|
+
return isRecord(current) ? current : undefined;
|
|
228
|
+
}
|
|
229
|
+
function openApiTestPoints(field, dimension, values = []) {
|
|
230
|
+
const prefix = `TP-${openApiToken(field)}`;
|
|
231
|
+
if (dimension === "enum")
|
|
232
|
+
return [...values.map((value) => `${prefix}-ENUM-${openApiToken(value)}`), `${prefix}-ENUM-UNKNOWN`, `${prefix}-ENUM-CASE-VARIANT`, `${prefix}-ENUM-WHITESPACE`, `${prefix}-ENUM-EMPTY`, `${prefix}-ENUM-NULL`, `${prefix}-ENUM-WRONG-TYPE`];
|
|
233
|
+
if (dimension === "boundary")
|
|
234
|
+
return [`${prefix}-MIN-1`, `${prefix}-MIN`, `${prefix}-NOMINAL`, `${prefix}-MAX`, `${prefix}-MAX-PLUS-1`];
|
|
235
|
+
if (dimension === "format")
|
|
236
|
+
return [`${prefix}-VALID-CLASS`, `${prefix}-VALID-MIXED`, `${prefix}-UPPERCASE`, `${prefix}-WHITESPACE`, `${prefix}-PUNCTUATION`, `${prefix}-SLASH`, `${prefix}-EMOJI`, `${prefix}-CONTROL`];
|
|
237
|
+
return [`${prefix}-PRESENT`, `${prefix}-MISSING`, `${prefix}-NULL`, `${prefix}-WRONG-TYPE`];
|
|
238
|
+
}
|
|
239
|
+
function resolveBoundSourcePath(workspaceRoot, taskId, sourcePath) {
|
|
240
|
+
if (path.isAbsolute(sourcePath))
|
|
241
|
+
return sourcePath;
|
|
242
|
+
const normalized = sourcePath.replaceAll("\\", "/").replace(/^\.\//, "");
|
|
243
|
+
const boundPrefix = `.harness/tasks/${taskId}/`;
|
|
244
|
+
return normalized.startsWith(boundPrefix)
|
|
245
|
+
? path.join(workspaceRoot, ...normalized.split("/"))
|
|
246
|
+
: path.join(workspaceRoot, ".harness", "tasks", taskId, ...normalized.split("/"));
|
|
247
|
+
}
|
|
248
|
+
export async function extractBackendTestOpenApiRules(input) {
|
|
249
|
+
const rules = [];
|
|
250
|
+
for (const referencePath of input.sourceBinding.referencePaths) {
|
|
251
|
+
const physicalPath = resolveBoundSourcePath(input.workspaceRoot, input.sourceBinding.taskId, referencePath);
|
|
252
|
+
let document;
|
|
253
|
+
try {
|
|
254
|
+
const parsed = YAML.parse(await readFile(physicalPath, "utf8"));
|
|
255
|
+
if (!isRecord(parsed) || !isRecord(parsed.paths))
|
|
256
|
+
continue;
|
|
257
|
+
document = parsed;
|
|
258
|
+
}
|
|
259
|
+
catch {
|
|
260
|
+
continue;
|
|
261
|
+
}
|
|
262
|
+
for (const [apiPath, pathItem] of Object.entries(document.paths)) {
|
|
263
|
+
if (!isRecord(pathItem))
|
|
264
|
+
continue;
|
|
265
|
+
for (const method of ["get", "post", "put", "patch", "delete"]) {
|
|
266
|
+
const operation = pathItem[method];
|
|
267
|
+
if (!isRecord(operation))
|
|
268
|
+
continue;
|
|
269
|
+
const operationBase = `API-${method.toUpperCase()}-${openApiToken(apiPath)}`;
|
|
270
|
+
const responses = isRecord(operation.responses) ? operation.responses : {};
|
|
271
|
+
const responseStatuses = Object.keys(responses).filter((status) => /^(?:[1-5]\d\d|default)$/i.test(status)).sort();
|
|
272
|
+
if (responseStatuses.length > 0) {
|
|
273
|
+
rules.push({
|
|
274
|
+
ruleKey: `${operationBase}-RESPONSE-STATUS`,
|
|
275
|
+
priority: "P1",
|
|
276
|
+
source: referencePath,
|
|
277
|
+
endpointField: `${method.toUpperCase()} ${apiPath} / response status`,
|
|
278
|
+
dimension: "response-status",
|
|
279
|
+
rule: `documented response statuses: ${responseStatuses.join(", ")}`,
|
|
280
|
+
requiredTestPoints: responseStatuses.map((status) => `TP-RESPONSE-STATUS-${openApiToken(status)}`),
|
|
281
|
+
});
|
|
282
|
+
}
|
|
283
|
+
const parameterValues = [
|
|
284
|
+
...(Array.isArray(pathItem.parameters) ? pathItem.parameters : []),
|
|
285
|
+
...(Array.isArray(operation.parameters) ? operation.parameters : []),
|
|
286
|
+
];
|
|
287
|
+
for (const rawParameter of parameterValues) {
|
|
288
|
+
const parameter = resolveOpenApiSchema(document, rawParameter);
|
|
289
|
+
if (!parameter || typeof parameter.name !== "string")
|
|
290
|
+
continue;
|
|
291
|
+
const field = parameter.name;
|
|
292
|
+
const property = resolveOpenApiSchema(document, parameter.schema);
|
|
293
|
+
if (!property)
|
|
294
|
+
continue;
|
|
295
|
+
const base = `${operationBase}-${openApiToken(field)}`;
|
|
296
|
+
const endpointField = `${method.toUpperCase()} ${apiPath} / ${String(parameter.in ?? "parameter")}:${field}`;
|
|
297
|
+
const add = (suffix, dimension, rule, points = openApiTestPoints(field, dimension)) => {
|
|
298
|
+
rules.push({ ruleKey: `${base}-${suffix}`, priority: "P1", source: referencePath, endpointField, dimension, rule, requiredTestPoints: points });
|
|
299
|
+
};
|
|
300
|
+
if (parameter.required === true)
|
|
301
|
+
add("REQUIRED", "requiredness", `${field} parameter is required`);
|
|
302
|
+
if (Array.isArray(property.enum)) {
|
|
303
|
+
const values = property.enum.map(String);
|
|
304
|
+
add("ENUM", "enum", `${field} enum: ${values.join(", ")}`, openApiTestPoints(field, "enum", values));
|
|
305
|
+
}
|
|
306
|
+
const boundaryPoints = openApiTestPoints(field, "boundary");
|
|
307
|
+
if (typeof property.minimum === "number")
|
|
308
|
+
add("MINIMUM", "boundary", `${field} minimum=${property.minimum}`, boundaryPoints.filter((point) => !point.includes("MAX")));
|
|
309
|
+
if (typeof property.maximum === "number")
|
|
310
|
+
add("MAXIMUM", "boundary", `${field} maximum=${property.maximum}`, boundaryPoints.filter((point) => !point.includes("MIN-")));
|
|
311
|
+
if (typeof property.pattern === "string")
|
|
312
|
+
add("PATTERN", "format", `${field} pattern=${property.pattern}`);
|
|
313
|
+
if (typeof property.format === "string")
|
|
314
|
+
add("FORMAT", "format", `${field} format=${property.format}`);
|
|
315
|
+
}
|
|
316
|
+
const requestBody = isRecord(operation.requestBody) ? operation.requestBody : undefined;
|
|
317
|
+
const content = requestBody && isRecord(requestBody.content) ? requestBody.content : undefined;
|
|
318
|
+
const media = content && (content["application/json"] ?? Object.values(content)[0]);
|
|
319
|
+
const schema = resolveOpenApiSchema(document, isRecord(media) ? media.schema : undefined);
|
|
320
|
+
if (!schema)
|
|
321
|
+
continue;
|
|
322
|
+
const required = new Set(Array.isArray(schema.required) ? schema.required.filter((value) => typeof value === "string") : []);
|
|
323
|
+
const properties = isRecord(schema.properties) ? schema.properties : {};
|
|
324
|
+
for (const [field, rawProperty] of Object.entries(properties)) {
|
|
325
|
+
const property = resolveOpenApiSchema(document, rawProperty);
|
|
326
|
+
if (!property)
|
|
327
|
+
continue;
|
|
328
|
+
const base = `${operationBase}-${openApiToken(field)}`;
|
|
329
|
+
const endpointField = `${method.toUpperCase()} ${apiPath} / ${field}`;
|
|
330
|
+
const add = (suffix, dimension, rule, points = openApiTestPoints(field, dimension)) => {
|
|
331
|
+
rules.push({ ruleKey: `${base}-${suffix}`, priority: "P1", source: referencePath, endpointField, dimension, rule, requiredTestPoints: points });
|
|
332
|
+
};
|
|
333
|
+
if (required.has(field))
|
|
334
|
+
add("REQUIRED", "requiredness", `${field} is required`);
|
|
335
|
+
if (Array.isArray(property.enum)) {
|
|
336
|
+
const values = property.enum.map(String);
|
|
337
|
+
add("ENUM", "enum", `${field} enum: ${values.join(", ")}`, openApiTestPoints(field, "enum", values));
|
|
338
|
+
}
|
|
339
|
+
const boundaryPoints = openApiTestPoints(field, "boundary");
|
|
340
|
+
if (typeof property.minLength === "number")
|
|
341
|
+
add("MIN-LENGTH", "boundary", `${field} minLength=${property.minLength}`, boundaryPoints.filter((point) => !point.includes("MAX")));
|
|
342
|
+
if (typeof property.maxLength === "number")
|
|
343
|
+
add("MAX-LENGTH", "boundary", `${field} maxLength=${property.maxLength}`, boundaryPoints.filter((point) => !point.includes("MIN-")));
|
|
344
|
+
if (typeof property.minimum === "number")
|
|
345
|
+
add("MINIMUM", "boundary", `${field} minimum=${property.minimum}`, boundaryPoints.filter((point) => !point.includes("MAX")));
|
|
346
|
+
if (typeof property.maximum === "number")
|
|
347
|
+
add("MAXIMUM", "boundary", `${field} maximum=${property.maximum}`, boundaryPoints.filter((point) => !point.includes("MIN-")));
|
|
348
|
+
if (typeof property.pattern === "string")
|
|
349
|
+
add("PATTERN", "format", `${field} pattern=${property.pattern}`);
|
|
350
|
+
if (typeof property.format === "string")
|
|
351
|
+
add("FORMAT", "format", `${field} format=${property.format}`);
|
|
352
|
+
}
|
|
353
|
+
}
|
|
354
|
+
}
|
|
355
|
+
}
|
|
356
|
+
return rules.sort((left, right) => left.ruleKey.localeCompare(right.ruleKey));
|
|
357
|
+
}
|
|
358
|
+
function orderedUnique(values) {
|
|
359
|
+
const result = [];
|
|
360
|
+
const seen = new Set();
|
|
361
|
+
for (const value of values) {
|
|
362
|
+
if (!value || seen.has(value))
|
|
363
|
+
continue;
|
|
364
|
+
seen.add(value);
|
|
365
|
+
result.push(value);
|
|
366
|
+
}
|
|
367
|
+
return result;
|
|
368
|
+
}
|
|
369
|
+
function canonicalCaseId(value) {
|
|
370
|
+
const upper = value.toUpperCase().replaceAll("_", "-").replace(/-+/g, "-");
|
|
371
|
+
return /^(BE-[A-Z0-9]+(?:-[A-Z0-9]+)*)-\d{3}$/.test(upper) ? upper : value.toUpperCase();
|
|
372
|
+
}
|
|
373
|
+
function caseIds(value) {
|
|
374
|
+
return orderedUnique((value.match(CASE_ID_IN_TEXT) ?? []).map(canonicalCaseId));
|
|
375
|
+
}
|
|
376
|
+
function sectionBody(body, aliases) {
|
|
377
|
+
const escaped = aliases.map((value) => value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")).join("|");
|
|
378
|
+
const marker = new RegExp(`^###\\s+(?:${escaped})\\s*$`, "mi");
|
|
379
|
+
const match = marker.exec(body);
|
|
380
|
+
if (!match)
|
|
381
|
+
return "";
|
|
382
|
+
const rest = body.slice(match.index + match[0].length);
|
|
383
|
+
const next = /^###\s+/m.exec(rest);
|
|
384
|
+
return rest.slice(0, next?.index ?? rest.length);
|
|
385
|
+
}
|
|
386
|
+
function listTokens(body, aliases, pattern) {
|
|
387
|
+
const section = sectionBody(body, aliases).replaceAll("`", "");
|
|
388
|
+
const source = pattern.source.replace(/^\^/, "").replace(/\$$/, "");
|
|
389
|
+
return orderedUnique(section.match(new RegExp(source, "g")) ?? []);
|
|
390
|
+
}
|
|
391
|
+
function scenarioTypes(body) {
|
|
392
|
+
return orderedUnique(sectionBody(body, CASE_SECTION_ALIASES.scenarioTypes)
|
|
393
|
+
.split(/\r?\n/)
|
|
394
|
+
.map((line) => line.replace(/^\s*(?:[-*+] |\d+[.)]\s*)/, "").replaceAll("`", "").trim())
|
|
395
|
+
.flatMap((line) => line.split(/[;,,、]/).map((item) => item.trim().toLowerCase()))
|
|
396
|
+
.filter(Boolean));
|
|
397
|
+
}
|
|
398
|
+
function declaredScripts(body) {
|
|
399
|
+
const automation = sectionBody(body, CASE_SECTION_ALIASES.automation);
|
|
400
|
+
return orderedUnique([...automation.matchAll(/testcase\/[A-Za-z0-9_./-]*test_[A-Za-z0-9_.-]*\.py/gi)]
|
|
401
|
+
.map((match) => match[0].replaceAll("\\", "/")));
|
|
402
|
+
}
|
|
403
|
+
function automationLabelBody(body, labels) {
|
|
404
|
+
const automation = sectionBody(body, CASE_SECTION_ALIASES.automation);
|
|
405
|
+
const escaped = labels.map((label) => label.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")).join("|");
|
|
406
|
+
const match = new RegExp(`^\\s*[-*+]\\s*(?:${escaped})\\s*[::]\\s*(.*)$`, "mi").exec(automation);
|
|
407
|
+
return match?.[1]?.replaceAll("`", "").trim() ?? "";
|
|
408
|
+
}
|
|
409
|
+
function testPointBindingFacts(body, testPoints) {
|
|
410
|
+
const bindings = TEST_POINT_BINDING_MODES.flatMap((mode) => {
|
|
411
|
+
const value = automationLabelBody(body, AUTOMATION_BINDING_LABELS[mode]);
|
|
412
|
+
return orderedUnique(value.match(/\bTP-[A-Z0-9]+(?:-[A-Z0-9]+)*\b/g) ?? [])
|
|
413
|
+
.map((testPoint) => ({ testPoint, mode }));
|
|
414
|
+
});
|
|
415
|
+
const counts = new Map();
|
|
416
|
+
for (const binding of bindings)
|
|
417
|
+
counts.set(binding.testPoint, (counts.get(binding.testPoint) ?? 0) + 1);
|
|
418
|
+
return {
|
|
419
|
+
testPointBindings: bindings,
|
|
420
|
+
unclassifiedTestPoints: testPoints.filter((testPoint) => !counts.has(testPoint)),
|
|
421
|
+
duplicateBindingTestPoints: testPoints.filter((testPoint) => (counts.get(testPoint) ?? 0) > 1),
|
|
422
|
+
};
|
|
423
|
+
}
|
|
424
|
+
function declaredPrimarySymbols(body) {
|
|
425
|
+
const value = automationLabelBody(body, ["primary symbol", "Primary Symbol", "主测试符号"]);
|
|
426
|
+
return orderedUnique(value.match(/\btest_[A-Za-z0-9_]+\b/g) ?? []);
|
|
427
|
+
}
|
|
428
|
+
function cleanTitle(heading, id) {
|
|
429
|
+
return heading.replace(/^##\s+/, "").replace(/^BE-[A-Z0-9_-]+-\d{2,3}\s*(?:[||—–-]\s*)?/i, "").trim() || id;
|
|
430
|
+
}
|
|
431
|
+
async function exists(filePath) {
|
|
432
|
+
try {
|
|
433
|
+
await access(filePath);
|
|
434
|
+
return true;
|
|
435
|
+
}
|
|
436
|
+
catch {
|
|
437
|
+
return false;
|
|
438
|
+
}
|
|
439
|
+
}
|
|
440
|
+
async function markdownModuleFiles(workspaceRoot) {
|
|
441
|
+
const dir = path.join(workspaceRoot, "testcase", "md");
|
|
442
|
+
const entries = await readdir(dir, { withFileTypes: true });
|
|
443
|
+
return entries
|
|
444
|
+
.filter((entry) => entry.isFile() && entry.name.toLowerCase().endsWith(".md") && entry.name.toLowerCase() !== "readme.md")
|
|
445
|
+
.map((entry) => path.join(dir, entry.name))
|
|
446
|
+
.sort();
|
|
447
|
+
}
|
|
448
|
+
async function parseMarkdownCases(workspaceRoot) {
|
|
449
|
+
const result = [];
|
|
450
|
+
for (const file of await markdownModuleFiles(workspaceRoot)) {
|
|
451
|
+
const markdown = await readFile(file, "utf8");
|
|
452
|
+
const headings = [...markdown.matchAll(CASE_HEADING)];
|
|
453
|
+
for (let index = 0; index < headings.length; index += 1) {
|
|
454
|
+
const heading = headings[index];
|
|
455
|
+
const id = canonicalCaseId(heading[1]);
|
|
456
|
+
const body = markdown.slice(heading.index, headings[index + 1]?.index ?? markdown.length);
|
|
457
|
+
const testPoints = listTokens(body, CASE_SECTION_ALIASES.testPoints, TEST_POINT);
|
|
458
|
+
result.push({
|
|
459
|
+
caseId: id,
|
|
460
|
+
title: cleanTitle(heading[0], id),
|
|
461
|
+
markdownPath: path.relative(workspaceRoot, file).replaceAll(path.sep, "/"),
|
|
462
|
+
acIds: orderedUnique(body.match(AC_ID_IN_TEXT) ?? []),
|
|
463
|
+
ruleKeys: listTokens(body, CASE_SECTION_ALIASES.rules, RULE_KEY),
|
|
464
|
+
testPoints,
|
|
465
|
+
...testPointBindingFacts(body, testPoints),
|
|
466
|
+
scenarioTypes: scenarioTypes(body),
|
|
467
|
+
declaredScripts: declaredScripts(body),
|
|
468
|
+
declaredPrimarySymbols: declaredPrimarySymbols(body),
|
|
469
|
+
body,
|
|
470
|
+
});
|
|
471
|
+
}
|
|
472
|
+
}
|
|
473
|
+
return result;
|
|
474
|
+
}
|
|
475
|
+
function parseTableRow(line) {
|
|
476
|
+
const trimmed = line.trim().replace(/^\|/, "").replace(/\|$/, "");
|
|
477
|
+
const cells = [];
|
|
478
|
+
let current = "";
|
|
479
|
+
for (let index = 0; index < trimmed.length; index += 1) {
|
|
480
|
+
const char = trimmed[index];
|
|
481
|
+
if (char === "\\" && trimmed[index + 1] === "|") {
|
|
482
|
+
current += "|";
|
|
483
|
+
index += 1;
|
|
484
|
+
}
|
|
485
|
+
else if (char === "|") {
|
|
486
|
+
cells.push(current.trim());
|
|
487
|
+
current = "";
|
|
488
|
+
}
|
|
489
|
+
else {
|
|
490
|
+
current += char;
|
|
491
|
+
}
|
|
492
|
+
}
|
|
493
|
+
cells.push(current.trim());
|
|
494
|
+
return cells;
|
|
495
|
+
}
|
|
496
|
+
function splitCellTokens(value, pattern) {
|
|
497
|
+
const values = orderedUnique(value.split(/;|<br\s*\/?\s*>|,|、/i).map((item) => item.replaceAll("`", "").trim()).filter(Boolean));
|
|
498
|
+
return pattern ? values.filter((item) => pattern.test(item)) : values;
|
|
499
|
+
}
|
|
500
|
+
function parseCoverageScope(readme) {
|
|
501
|
+
const findings = [];
|
|
502
|
+
const fallback = {
|
|
503
|
+
changeClassification: "new-operation",
|
|
504
|
+
coveragePolicy: "full-contract",
|
|
505
|
+
affectedOperations: [],
|
|
506
|
+
affectedRuleKeys: [],
|
|
507
|
+
regressionFloor: [],
|
|
508
|
+
scopeEvidence: [],
|
|
509
|
+
completenessClaim: "affected-operations-full",
|
|
510
|
+
};
|
|
511
|
+
const marker = /^##\s+Coverage Scope\s*$/mi.exec(readme);
|
|
512
|
+
if (!marker)
|
|
513
|
+
return { scope: fallback, findings: ["README is missing required ## Coverage Scope section; defaulted to full-contract"] };
|
|
514
|
+
const rest = readme.slice(marker.index + marker[0].length);
|
|
515
|
+
const nextHeading = /^##\s+/m.exec(rest);
|
|
516
|
+
const section = rest.slice(0, nextHeading?.index ?? rest.length);
|
|
517
|
+
const lines = section.split(/\r?\n/).filter((line) => line.trim().startsWith("|"));
|
|
518
|
+
if (lines.length < 2 || JSON.stringify(parseTableRow(lines[0])) !== JSON.stringify(["Field", "Value"])) {
|
|
519
|
+
return { scope: fallback, findings: ["Coverage Scope table headers must be exactly: Field | Value"] };
|
|
520
|
+
}
|
|
521
|
+
const separatorCells = lines[1] ? parseTableRow(lines[1]) : [];
|
|
522
|
+
const hasSeparator = separatorCells.length === 2 && separatorCells.every((cell) => /^:?-{3,}:?$/.test(cell));
|
|
523
|
+
if (!hasSeparator)
|
|
524
|
+
findings.push("Coverage Scope table is missing the required |---|---| separator row");
|
|
525
|
+
const values = new Map();
|
|
526
|
+
for (const line of lines.slice(hasSeparator ? 2 : 1)) {
|
|
527
|
+
const cells = parseTableRow(line);
|
|
528
|
+
if (cells.length !== 2) {
|
|
529
|
+
findings.push(`Coverage Scope row must have exactly 2 columns: ${line.slice(0, 160)}`);
|
|
530
|
+
continue;
|
|
531
|
+
}
|
|
532
|
+
const field = cells[0].trim();
|
|
533
|
+
if (values.has(field))
|
|
534
|
+
findings.push(`duplicate Coverage Scope field: ${field}`);
|
|
535
|
+
values.set(field, cells[1].trim());
|
|
536
|
+
}
|
|
537
|
+
const classificationRaw = (values.get("Change Classification") ?? "").replaceAll("`", "").trim();
|
|
538
|
+
const policyRaw = (values.get("Coverage Policy") ?? "").replaceAll("`", "").trim();
|
|
539
|
+
const changeClassification = CHANGE_CLASSIFICATIONS.includes(classificationRaw)
|
|
540
|
+
? classificationRaw
|
|
541
|
+
: fallback.changeClassification;
|
|
542
|
+
const coveragePolicy = COVERAGE_POLICIES.includes(policyRaw)
|
|
543
|
+
? policyRaw
|
|
544
|
+
: fallback.coveragePolicy;
|
|
545
|
+
if (!CHANGE_CLASSIFICATIONS.includes(classificationRaw))
|
|
546
|
+
findings.push(`invalid or missing Change Classification: ${classificationRaw || "<empty>"}`);
|
|
547
|
+
if (!COVERAGE_POLICIES.includes(policyRaw))
|
|
548
|
+
findings.push(`invalid or missing Coverage Policy: ${policyRaw || "<empty>"}`);
|
|
549
|
+
const expectedPolicy = COVERAGE_POLICY_BY_CLASSIFICATION[changeClassification];
|
|
550
|
+
if (coveragePolicy !== expectedPolicy)
|
|
551
|
+
findings.push(`Change Classification ${changeClassification} requires Coverage Policy ${expectedPolicy}, received ${coveragePolicy}`);
|
|
552
|
+
const affectedOperations = splitCellTokens(values.get("Affected Operations") ?? "").map((value) => value.replace(/\s+/g, " ").trim());
|
|
553
|
+
const affectedRuleKeys = splitCellTokens(values.get("Affected Rule Keys") ?? "", RULE_KEY);
|
|
554
|
+
const regressionFloor = splitCellTokens(values.get("Regression Floor") ?? "").map((value) => value.toLowerCase());
|
|
555
|
+
const scopeEvidence = splitCellTokens(values.get("Scope Evidence") ?? "");
|
|
556
|
+
if (affectedOperations.length === 0)
|
|
557
|
+
findings.push("Coverage Scope requires at least one Affected Operations entry");
|
|
558
|
+
if (affectedRuleKeys.length === 0)
|
|
559
|
+
findings.push("Coverage Scope requires at least one Affected Rule Keys entry");
|
|
560
|
+
if (scopeEvidence.length === 0)
|
|
561
|
+
findings.push("Coverage Scope requires non-empty Scope Evidence");
|
|
562
|
+
for (const required of REQUIRED_REGRESSION_FLOOR[changeClassification]) {
|
|
563
|
+
if (!regressionFloor.includes(required))
|
|
564
|
+
findings.push(`Coverage Scope regression floor is missing required entry: ${required}`);
|
|
565
|
+
}
|
|
566
|
+
return {
|
|
567
|
+
scope: {
|
|
568
|
+
changeClassification,
|
|
569
|
+
coveragePolicy,
|
|
570
|
+
affectedOperations,
|
|
571
|
+
affectedRuleKeys,
|
|
572
|
+
regressionFloor,
|
|
573
|
+
scopeEvidence,
|
|
574
|
+
completenessClaim: coveragePolicy === "full-contract" ? "affected-operations-full" : "affected-scope",
|
|
575
|
+
},
|
|
576
|
+
findings,
|
|
577
|
+
};
|
|
578
|
+
}
|
|
579
|
+
function operationFromEndpointField(endpointField) {
|
|
580
|
+
const match = /^([A-Z]+)\s+(\/\S+)/.exec(endpointField.trim());
|
|
581
|
+
return match ? `${match[1]} ${match[2]}` : undefined;
|
|
582
|
+
}
|
|
583
|
+
function parseCoverageMatrix(readme) {
|
|
584
|
+
const findings = [];
|
|
585
|
+
const marker = /^##\s+(?:(?:\d+(?:\.\d+)*[.)]?\s+)?(?:Coverage Matrix|覆盖矩阵))\s*$/mi.exec(readme);
|
|
586
|
+
if (!marker)
|
|
587
|
+
return { rows: [], findings: ["README is missing required ## Coverage Matrix section"] };
|
|
588
|
+
if (!/^##\s+Coverage Matrix\s*$/i.test(marker[0])) {
|
|
589
|
+
findings.push(`Coverage Matrix should use the exact canonical heading \"## Coverage Matrix\"; accepted compatibility heading: ${marker[0].trim()}`);
|
|
590
|
+
}
|
|
591
|
+
const rest = readme.slice(marker.index + marker[0].length);
|
|
592
|
+
const nextHeading = /^##\s+/m.exec(rest);
|
|
593
|
+
const section = rest.slice(0, nextHeading?.index ?? rest.length);
|
|
594
|
+
const lines = section.split(/\r?\n/).filter((line) => line.trim().startsWith("|"));
|
|
595
|
+
if (lines.length < 2)
|
|
596
|
+
return { rows: [], findings: ["Coverage Matrix table is missing or incomplete"] };
|
|
597
|
+
const headers = parseTableRow(lines[0]);
|
|
598
|
+
if (JSON.stringify(headers) !== JSON.stringify(COVERAGE_HEADERS)) {
|
|
599
|
+
findings.push(`Coverage Matrix headers must be exactly: ${COVERAGE_HEADERS.join(" | ")}`);
|
|
600
|
+
return { rows: [], findings };
|
|
601
|
+
}
|
|
602
|
+
const rows = [];
|
|
603
|
+
for (const line of lines.slice(2)) {
|
|
604
|
+
const cells = parseTableRow(line);
|
|
605
|
+
if (cells.length !== COVERAGE_HEADERS.length) {
|
|
606
|
+
findings.push(`Coverage Matrix row has ${cells.length} columns instead of ${COVERAGE_HEADERS.length}: ${line.slice(0, 160)}`);
|
|
607
|
+
continue;
|
|
608
|
+
}
|
|
609
|
+
rows.push(Object.fromEntries(COVERAGE_HEADERS.map((header, index) => [header, cells[index] ?? ""])));
|
|
610
|
+
}
|
|
611
|
+
if (rows.length === 0)
|
|
612
|
+
findings.push("Coverage Matrix contains no rule rows");
|
|
613
|
+
return { rows, findings };
|
|
614
|
+
}
|
|
615
|
+
async function fileHash(filePath) {
|
|
616
|
+
return createHash("sha256").update(await readFile(filePath)).digest("hex");
|
|
617
|
+
}
|
|
618
|
+
async function inputFileFacts(workspaceRoot, files) {
|
|
619
|
+
const result = [];
|
|
620
|
+
for (const file of orderedUnique(files).sort()) {
|
|
621
|
+
result.push({ path: path.relative(workspaceRoot, file).replaceAll(path.sep, "/"), sha256: await fileHash(file) });
|
|
622
|
+
}
|
|
623
|
+
return result;
|
|
624
|
+
}
|
|
625
|
+
function dimensionCounts(rules, matcher) {
|
|
626
|
+
const selected = rules.filter((rule) => matcher.test(rule.dimension));
|
|
627
|
+
return [
|
|
628
|
+
selected.reduce((sum, rule) => sum + rule.requiredTestPoints.length, 0),
|
|
629
|
+
selected.reduce((sum, rule) => sum + rule.coveredTestPoints.length, 0),
|
|
630
|
+
];
|
|
631
|
+
}
|
|
632
|
+
function renderCoverageTable(rules) {
|
|
633
|
+
return [
|
|
634
|
+
"| Rule Key | Priority | Dimension | Required | Covered | Missing | Status | Case IDs |",
|
|
635
|
+
"|---|---|---|---:|---:|---|---|---|",
|
|
636
|
+
...rules.map((rule) => `| ${rule.ruleKey} | ${rule.priority} | ${rule.dimension} | ${rule.requiredTestPoints.length} | ${rule.coveredTestPoints.length} | ${rule.missingTestPoints.join(", ") || "—"} | ${rule.status} | ${rule.caseIds.join(", ") || "—"} |`),
|
|
637
|
+
];
|
|
638
|
+
}
|
|
639
|
+
export async function analyzeBackendTestCaseCoverage(input) {
|
|
640
|
+
const findings = [];
|
|
641
|
+
const readmePath = path.join(input.workspaceRoot, "testcase", "md", "README.md");
|
|
642
|
+
const readme = (await exists(readmePath)) ? await readFile(readmePath, "utf8") : "";
|
|
643
|
+
const coverageScope = parseCoverageScope(readme);
|
|
644
|
+
findings.push(...coverageScope.findings);
|
|
645
|
+
const matrix = parseCoverageMatrix(readme);
|
|
646
|
+
findings.push(...matrix.findings);
|
|
647
|
+
const cases = await parseMarkdownCases(input.workspaceRoot);
|
|
648
|
+
for (const file of await markdownModuleFiles(input.workspaceRoot)) {
|
|
649
|
+
const markdown = await readFile(file, "utf8");
|
|
650
|
+
for (const match of markdown.matchAll(NON_CANONICAL_CASE_HEADING)) {
|
|
651
|
+
findings.push(`non-canonical Case ID heading must use BE-<MODULE>-<NNN>: ${match[1]}`);
|
|
652
|
+
}
|
|
653
|
+
}
|
|
654
|
+
const caseById = new Map(cases.map((item) => [item.caseId, item]));
|
|
655
|
+
const rules = [];
|
|
656
|
+
const evidenceGaps = [];
|
|
657
|
+
const conflicts = [];
|
|
658
|
+
const seenRules = new Set();
|
|
659
|
+
for (const row of matrix.rows) {
|
|
660
|
+
const ruleKey = row["Rule Key"].replaceAll("`", "").trim();
|
|
661
|
+
const priority = row.Priority.trim().toUpperCase();
|
|
662
|
+
const declaredStatus = row.Status.trim().toUpperCase();
|
|
663
|
+
if (!RULE_KEY.test(ruleKey))
|
|
664
|
+
findings.push(`invalid Rule Key: ${ruleKey || "<empty>"}`);
|
|
665
|
+
if (seenRules.has(ruleKey))
|
|
666
|
+
findings.push(`duplicate Coverage Matrix Rule Key: ${ruleKey}`);
|
|
667
|
+
seenRules.add(ruleKey);
|
|
668
|
+
if (!/^(?:P0|P1|P2)$/.test(priority))
|
|
669
|
+
findings.push(`${ruleKey} has invalid priority: ${priority}`);
|
|
670
|
+
if (!/^(?:COVERED|PARTIAL|GAP|CONFLICT)$/.test(declaredStatus))
|
|
671
|
+
findings.push(`${ruleKey} has invalid status: ${declaredStatus}`);
|
|
672
|
+
const requiredTestPoints = splitCellTokens(row["Required Test Points"], TEST_POINT);
|
|
673
|
+
const rowCaseIds = splitCellTokens(row["Case IDs"]).map(canonicalCaseId);
|
|
674
|
+
if (requiredTestPoints.length === 0 && declaredStatus === "COVERED")
|
|
675
|
+
findings.push(`${ruleKey} declares COVERED without Required Test Points`);
|
|
676
|
+
const declaredPoints = orderedUnique(rowCaseIds.flatMap((id) => caseById.get(id)?.testPoints ?? []));
|
|
677
|
+
const coveredTestPoints = requiredTestPoints.filter((point) => declaredPoints.includes(point));
|
|
678
|
+
const missingTestPoints = requiredTestPoints.filter((point) => !declaredPoints.includes(point));
|
|
679
|
+
for (const id of rowCaseIds) {
|
|
680
|
+
const testCase = caseById.get(id);
|
|
681
|
+
if (!testCase)
|
|
682
|
+
findings.push(`${ruleKey} references missing Markdown Case: ${id}`);
|
|
683
|
+
else if (!testCase.ruleKeys.includes(ruleKey))
|
|
684
|
+
findings.push(`${id} does not reference Coverage Matrix rule ${ruleKey}`);
|
|
685
|
+
}
|
|
686
|
+
for (const testCase of cases.filter((item) => item.ruleKeys.includes(ruleKey))) {
|
|
687
|
+
if (!rowCaseIds.includes(testCase.caseId))
|
|
688
|
+
findings.push(`${testCase.caseId} references ${ruleKey} but is absent from its Coverage Matrix Case IDs`);
|
|
689
|
+
}
|
|
690
|
+
let status = declaredStatus;
|
|
691
|
+
if (status === "COVERED" && missingTestPoints.length > 0)
|
|
692
|
+
status = "PARTIAL";
|
|
693
|
+
if (missingTestPoints.length > 0)
|
|
694
|
+
findings.push(`${ruleKey} is missing required test points: ${missingTestPoints.join(", ")}`);
|
|
695
|
+
if (status === "GAP")
|
|
696
|
+
evidenceGaps.push(`${ruleKey}: ${row.Rule}`);
|
|
697
|
+
if (status === "CONFLICT")
|
|
698
|
+
conflicts.push(`${ruleKey}: ${row.Rule}`);
|
|
699
|
+
if (status !== "COVERED")
|
|
700
|
+
findings.push(`${ruleKey} coverage status is ${status}`);
|
|
701
|
+
rules.push({
|
|
702
|
+
ruleKey,
|
|
703
|
+
priority: /^(?:P0|P1|P2)$/.test(priority) ? priority : "P2",
|
|
704
|
+
source: row.Source || "unavailable",
|
|
705
|
+
endpointField: row["Endpoint/Field"] || "unavailable",
|
|
706
|
+
dimension: row.Dimension.trim().toLowerCase() || "other",
|
|
707
|
+
rule: row.Rule || "unavailable",
|
|
708
|
+
requiredTestPoints,
|
|
709
|
+
caseIds: rowCaseIds,
|
|
710
|
+
declaredStatus: /^(?:COVERED|PARTIAL|GAP|CONFLICT)$/.test(declaredStatus) ? declaredStatus : "PARTIAL",
|
|
711
|
+
coveredTestPoints,
|
|
712
|
+
missingTestPoints,
|
|
713
|
+
status,
|
|
714
|
+
});
|
|
715
|
+
}
|
|
716
|
+
const extractedOpenApiRules = await extractBackendTestOpenApiRules(input);
|
|
717
|
+
const knownOperations = orderedUnique(extractedOpenApiRules.map((rule) => operationFromEndpointField(rule.endpointField)).filter((value) => Boolean(value)));
|
|
718
|
+
for (const affectedOperation of coverageScope.scope.affectedOperations) {
|
|
719
|
+
if (knownOperations.length > 0 && !knownOperations.includes(affectedOperation))
|
|
720
|
+
findings.push(`Coverage Scope Affected Operation is absent from bound OpenAPI evidence: ${affectedOperation}`);
|
|
721
|
+
}
|
|
722
|
+
const requiredOpenApiRules = extractedOpenApiRules.filter((rule) => {
|
|
723
|
+
const operation = operationFromEndpointField(rule.endpointField);
|
|
724
|
+
return Boolean(operation && coverageScope.scope.affectedOperations.includes(operation));
|
|
725
|
+
});
|
|
726
|
+
for (const extracted of requiredOpenApiRules) {
|
|
727
|
+
if (seenRules.has(extracted.ruleKey))
|
|
728
|
+
continue;
|
|
729
|
+
findings.push(`Coverage Matrix is missing documented in-scope OpenAPI rule: ${extracted.ruleKey} (${extracted.source})`);
|
|
730
|
+
seenRules.add(extracted.ruleKey);
|
|
731
|
+
rules.push({
|
|
732
|
+
...extracted,
|
|
733
|
+
caseIds: [],
|
|
734
|
+
declaredStatus: "PARTIAL",
|
|
735
|
+
coveredTestPoints: [],
|
|
736
|
+
missingTestPoints: extracted.requiredTestPoints,
|
|
737
|
+
status: "PARTIAL",
|
|
738
|
+
});
|
|
739
|
+
}
|
|
740
|
+
for (const affectedRuleKey of coverageScope.scope.affectedRuleKeys) {
|
|
741
|
+
if (!seenRules.has(affectedRuleKey))
|
|
742
|
+
findings.push(`Coverage Scope Affected Rule Key is absent from Coverage Matrix: ${affectedRuleKey}`);
|
|
743
|
+
}
|
|
744
|
+
const nonCrossCuttingOwners = new Map();
|
|
745
|
+
for (const testCase of cases) {
|
|
746
|
+
if (testCase.ruleKeys.length === 0)
|
|
747
|
+
findings.push(`${testCase.caseId} missing section or entries: 覆盖规则`);
|
|
748
|
+
if (testCase.testPoints.length === 0)
|
|
749
|
+
findings.push(`${testCase.caseId} missing section or entries: 测试点`);
|
|
750
|
+
if (testCase.scenarioTypes.length === 0)
|
|
751
|
+
findings.push(`${testCase.caseId} missing section or entries: 场景类型`);
|
|
752
|
+
if (testCase.declaredPrimarySymbols.length !== 1)
|
|
753
|
+
findings.push(`${testCase.caseId} must declare exactly one primary symbol`);
|
|
754
|
+
if (testCase.unclassifiedTestPoints.length > 0)
|
|
755
|
+
findings.push(`${testCase.caseId} has unclassified Test Points: ${testCase.unclassifiedTestPoints.join(", ")}`);
|
|
756
|
+
if (testCase.duplicateBindingTestPoints.length > 0)
|
|
757
|
+
findings.push(`${testCase.caseId} has duplicate Test Point bindings: ${testCase.duplicateBindingTestPoints.join(", ")}`);
|
|
758
|
+
const extraBindings = testCase.testPointBindings.filter((binding) => !testCase.testPoints.includes(binding.testPoint));
|
|
759
|
+
if (extraBindings.length > 0)
|
|
760
|
+
findings.push(`${testCase.caseId} binds undeclared Test Points: ${orderedUnique(extraBindings.map((binding) => binding.testPoint)).join(", ")}`);
|
|
761
|
+
for (const binding of testCase.testPointBindings.filter((item) => item.mode !== "cross-cutting")) {
|
|
762
|
+
const owners = nonCrossCuttingOwners.get(binding.testPoint) ?? [];
|
|
763
|
+
owners.push(testCase.caseId);
|
|
764
|
+
nonCrossCuttingOwners.set(binding.testPoint, owners);
|
|
765
|
+
}
|
|
766
|
+
for (const ruleKey of testCase.ruleKeys)
|
|
767
|
+
if (!seenRules.has(ruleKey))
|
|
768
|
+
findings.push(`${testCase.caseId} references unknown Rule Key: ${ruleKey}`);
|
|
769
|
+
}
|
|
770
|
+
for (const [testPoint, owners] of nonCrossCuttingOwners) {
|
|
771
|
+
const uniqueOwners = orderedUnique(owners);
|
|
772
|
+
if (uniqueOwners.length > 1)
|
|
773
|
+
findings.push(`${testPoint} is bound as a non-cross-cutting Test Point by multiple Cases: ${uniqueOwners.join(", ")}`);
|
|
774
|
+
}
|
|
775
|
+
const explicitAc = input.sourceBinding.requirementIds.filter((id) => id.startsWith("AC-"));
|
|
776
|
+
const coveredAc = new Set(cases.flatMap((item) => item.acIds));
|
|
777
|
+
const [enumValueCount, coveredEnumValueCount] = dimensionCounts(rules, /enum/);
|
|
778
|
+
const [boundaryPointCount, coveredBoundaryPointCount] = dimensionCounts(rules, /boundary|length|numeric/);
|
|
779
|
+
const [formatClassCount, coveredFormatClassCount] = dimensionCounts(rules, /format|pattern|charset/);
|
|
780
|
+
const [businessStateCount, coveredBusinessStateCount] = dimensionCounts(rules, /business-state|state-transition|uniqueness|lifecycle/);
|
|
781
|
+
const invalidPointPattern = /(?:UNKNOWN|CASE-VARIANT|WHITESPACE|EMPTY|NULL|WRONG-TYPE|MIN-1|MAX-PLUS-1|UPPERCASE|PUNCTUATION|SLASH|EMOJI|CONTROL)$/;
|
|
782
|
+
const invalidEquivalenceClassCount = rules.reduce((sum, rule) => sum + rule.requiredTestPoints.filter((point) => invalidPointPattern.test(point)).length, 0);
|
|
783
|
+
const coveredInvalidEquivalenceClassCount = rules.reduce((sum, rule) => sum + rule.coveredTestPoints.filter((point) => invalidPointPattern.test(point)).length, 0);
|
|
784
|
+
const facts = backendTestCaseCoverageFactsSchema.parse({
|
|
785
|
+
schemaId: "backend-test-case-coverage-facts-v3",
|
|
786
|
+
schemaVersion: 3,
|
|
787
|
+
taskId: input.sourceBinding.taskId,
|
|
788
|
+
status: findings.length === 0 ? "PASS" : "FAIL",
|
|
789
|
+
sourceBinding: input.sourceBinding,
|
|
790
|
+
inputFiles: await inputFileFacts(input.workspaceRoot, [
|
|
791
|
+
...((await exists(readmePath)) ? [readmePath] : []),
|
|
792
|
+
...(await markdownModuleFiles(input.workspaceRoot)),
|
|
793
|
+
...(await Promise.all(input.sourceBinding.referencePaths.map(async (referencePath) => {
|
|
794
|
+
const physicalPath = resolveBoundSourcePath(input.workspaceRoot, input.sourceBinding.taskId, referencePath);
|
|
795
|
+
return (await exists(physicalPath)) ? physicalPath : "";
|
|
796
|
+
}))).filter(Boolean),
|
|
797
|
+
]),
|
|
798
|
+
coverageScope: coverageScope.scope,
|
|
799
|
+
summary: {
|
|
800
|
+
explicitAcCount: explicitAc.length,
|
|
801
|
+
coveredAcCount: explicitAc.filter((id) => coveredAc.has(id)).length,
|
|
802
|
+
ruleCount: rules.length,
|
|
803
|
+
coveredRuleCount: rules.filter((rule) => rule.status === "COVERED").length,
|
|
804
|
+
testPointCount: rules.reduce((sum, rule) => sum + rule.requiredTestPoints.length, 0),
|
|
805
|
+
coveredTestPointCount: rules.reduce((sum, rule) => sum + rule.coveredTestPoints.length, 0),
|
|
806
|
+
enumValueCount,
|
|
807
|
+
coveredEnumValueCount,
|
|
808
|
+
invalidEquivalenceClassCount,
|
|
809
|
+
coveredInvalidEquivalenceClassCount,
|
|
810
|
+
boundaryPointCount,
|
|
811
|
+
coveredBoundaryPointCount,
|
|
812
|
+
formatClassCount,
|
|
813
|
+
coveredFormatClassCount,
|
|
814
|
+
businessStateCount,
|
|
815
|
+
coveredBusinessStateCount,
|
|
816
|
+
gapCount: evidenceGaps.length,
|
|
817
|
+
conflictCount: conflicts.length,
|
|
818
|
+
variantTestPointCount: cases.reduce((sum, item) => sum + item.testPointBindings.filter((binding) => binding.mode === "variant").length, 0),
|
|
819
|
+
assertionTestPointCount: cases.reduce((sum, item) => sum + item.testPointBindings.filter((binding) => binding.mode === "assertion").length, 0),
|
|
820
|
+
crossCuttingTestPointCount: cases.reduce((sum, item) => sum + item.testPointBindings.filter((binding) => binding.mode === "cross-cutting").length, 0),
|
|
821
|
+
unclassifiedTestPointCount: cases.reduce((sum, item) => sum + item.unclassifiedTestPoints.length, 0),
|
|
822
|
+
duplicateBindingTestPointCount: cases.reduce((sum, item) => sum + item.duplicateBindingTestPoints.length, 0),
|
|
823
|
+
},
|
|
824
|
+
cases: cases.map(({ body: _body, ...item }) => item),
|
|
825
|
+
rules,
|
|
826
|
+
findings: orderedUnique(findings),
|
|
827
|
+
evidenceGaps,
|
|
828
|
+
conflicts,
|
|
829
|
+
});
|
|
830
|
+
const summary = facts.summary;
|
|
831
|
+
const markdown = [
|
|
832
|
+
"# Backend Test Case Coverage Analysis",
|
|
833
|
+
"",
|
|
834
|
+
"## Status",
|
|
835
|
+
"",
|
|
836
|
+
facts.status,
|
|
837
|
+
"",
|
|
838
|
+
"## Coverage Scope",
|
|
839
|
+
"",
|
|
840
|
+
`- Change Classification: ${facts.coverageScope.changeClassification}`,
|
|
841
|
+
`- Coverage Policy: ${facts.coverageScope.coveragePolicy}`,
|
|
842
|
+
`- Completeness Claim: ${facts.coverageScope.completenessClaim}`,
|
|
843
|
+
`- Affected Operations: ${facts.coverageScope.affectedOperations.join(", ") || "Unavailable"}`,
|
|
844
|
+
`- Affected Rule Keys: ${facts.coverageScope.affectedRuleKeys.join(", ") || "Unavailable"}`,
|
|
845
|
+
`- Regression Floor: ${facts.coverageScope.regressionFloor.join(", ") || "Unavailable"}`,
|
|
846
|
+
`- Scope Evidence: ${facts.coverageScope.scopeEvidence.join(", ") || "Unavailable"}`,
|
|
847
|
+
...(facts.coverageScope.completenessClaim === "affected-scope" ? ["- Coverage is scoped to affected operations; it is not whole-API completeness."] : ["- Coverage fully evaluates the declared affected operations; it is not whole-API completeness unless every operation is explicitly listed."]),
|
|
848
|
+
"",
|
|
849
|
+
"## Coverage Summary",
|
|
850
|
+
"",
|
|
851
|
+
"| Dimension | Total | Covered |",
|
|
852
|
+
"|---|---:|---:|",
|
|
853
|
+
`| Product AC | ${summary.explicitAcCount} | ${summary.coveredAcCount} |`,
|
|
854
|
+
`| Rules | ${summary.ruleCount} | ${summary.coveredRuleCount} |`,
|
|
855
|
+
`| Required Test Points | ${summary.testPointCount} | ${summary.coveredTestPointCount} |`,
|
|
856
|
+
`| Enum | ${summary.enumValueCount} | ${summary.coveredEnumValueCount} |`,
|
|
857
|
+
`| Boundary | ${summary.boundaryPointCount} | ${summary.coveredBoundaryPointCount} |`,
|
|
858
|
+
`| Format | ${summary.formatClassCount} | ${summary.coveredFormatClassCount} |`,
|
|
859
|
+
`| Business State | ${summary.businessStateCount} | ${summary.coveredBusinessStateCount} |`,
|
|
860
|
+
`| Variant bindings | ${summary.variantTestPointCount} | ${summary.variantTestPointCount} |`,
|
|
861
|
+
`| Assertion bindings | ${summary.assertionTestPointCount} | ${summary.assertionTestPointCount} |`,
|
|
862
|
+
`| Cross-cutting bindings | ${summary.crossCuttingTestPointCount} | ${summary.crossCuttingTestPointCount} |`,
|
|
863
|
+
`| Unclassified bindings | ${summary.unclassifiedTestPointCount} | 0 |`,
|
|
864
|
+
`| Duplicate bindings | ${summary.duplicateBindingTestPointCount} | 0 |`,
|
|
865
|
+
"",
|
|
866
|
+
"## Product Requirement Coverage",
|
|
867
|
+
"",
|
|
868
|
+
...renderCoverageTable(rules.filter((rule) => rule.priority === "P0" || rule.ruleKey.startsWith("AC-"))),
|
|
869
|
+
"",
|
|
870
|
+
"## API Operation and Field Rule Coverage",
|
|
871
|
+
"",
|
|
872
|
+
...renderCoverageTable(rules.filter((rule) => rule.priority !== "P0")),
|
|
873
|
+
"",
|
|
874
|
+
"## Business State Coverage",
|
|
875
|
+
"",
|
|
876
|
+
...renderCoverageTable(rules.filter((rule) => /business-state|state-transition|uniqueness|lifecycle/.test(rule.dimension))),
|
|
877
|
+
"",
|
|
878
|
+
"## Enum Coverage",
|
|
879
|
+
"",
|
|
880
|
+
...renderCoverageTable(rules.filter((rule) => /enum/.test(rule.dimension))),
|
|
881
|
+
"",
|
|
882
|
+
"## Boundary and Format Coverage",
|
|
883
|
+
"",
|
|
884
|
+
...renderCoverageTable(rules.filter((rule) => /boundary|length|numeric|format|pattern|charset/.test(rule.dimension))),
|
|
885
|
+
"",
|
|
886
|
+
"## Uncovered/Partial Rules",
|
|
887
|
+
"",
|
|
888
|
+
...(rules.filter((rule) => rule.status !== "COVERED").map((rule) => `- ${rule.ruleKey}: ${rule.status}; missing=${rule.missingTestPoints.join(", ") || "none"}`)),
|
|
889
|
+
...(rules.every((rule) => rule.status === "COVERED") ? ["- None"] : []),
|
|
890
|
+
"",
|
|
891
|
+
"## Conflicts and Evidence Gaps",
|
|
892
|
+
"",
|
|
893
|
+
...([...evidenceGaps, ...conflicts].length ? [...evidenceGaps, ...conflicts].map((item) => `- ${item}`) : ["- None"]),
|
|
894
|
+
"",
|
|
895
|
+
"## Matrix/Case Consistency Findings",
|
|
896
|
+
"",
|
|
897
|
+
...(facts.findings.length ? facts.findings.map((item) => `- ${item}`) : ["- None"]),
|
|
898
|
+
"",
|
|
899
|
+
].join("\n");
|
|
900
|
+
return { facts, markdown };
|
|
901
|
+
}
|
|
902
|
+
function normalizeModuleStem(markdownPath) {
|
|
903
|
+
return path.posix.basename(markdownPath).replace(/\.md$/i, "").toLowerCase().replace(/[^a-z0-9]+/g, "_").replace(/^_+|_+$/g, "").replace(/_+/g, "_");
|
|
904
|
+
}
|
|
905
|
+
function expectedScript(markdownPath) {
|
|
906
|
+
return `testcase/test_${normalizeModuleStem(markdownPath)}.py`;
|
|
907
|
+
}
|
|
908
|
+
function symbolCaseId(symbol) {
|
|
909
|
+
const match = symbol.match(/^test_(BE(?:_[A-Z0-9]+)+?_\d{2,3})(?:_|$)/i);
|
|
910
|
+
return match ? canonicalCaseId(match[1].replaceAll("_", "-")) : undefined;
|
|
911
|
+
}
|
|
912
|
+
function metadataTestPoints(region, label) {
|
|
913
|
+
const escaped = label.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
914
|
+
const match = new RegExp(`^\\s*${escaped}\\s*:\\s*(.*)$`, "mi").exec(region);
|
|
915
|
+
return orderedUnique(match?.[1]?.match(/\bTP-[A-Z0-9]+(?:-[A-Z0-9]+)*\b/g) ?? []);
|
|
916
|
+
}
|
|
917
|
+
function pytestFunctionRegionStart(source, functionIndex) {
|
|
918
|
+
let regionStart = source.lastIndexOf("\n", functionIndex - 1) + 1;
|
|
919
|
+
let cursor = regionStart;
|
|
920
|
+
while (cursor > 0) {
|
|
921
|
+
const previousEnd = cursor - 1;
|
|
922
|
+
const previousStart = source.lastIndexOf("\n", previousEnd - 1) + 1;
|
|
923
|
+
const previous = source.slice(previousStart, previousEnd).replace(/\r$/, "");
|
|
924
|
+
if (!previous.trim())
|
|
925
|
+
break;
|
|
926
|
+
regionStart = previousStart;
|
|
927
|
+
cursor = previousStart;
|
|
928
|
+
}
|
|
929
|
+
return regionStart;
|
|
930
|
+
}
|
|
931
|
+
function pytestParameterCollections(source) {
|
|
932
|
+
const collections = new Map();
|
|
933
|
+
for (const match of source.matchAll(/^([A-Z][A-Z0-9_]*)\s*=\s*\[([\s\S]*?)^\]/gm)) {
|
|
934
|
+
collections.set(match[1], orderedUnique([...match[2].matchAll(/\bid\s*=\s*["'](TP-[A-Z0-9-]+)["']/g)].map((item) => item[1])));
|
|
935
|
+
}
|
|
936
|
+
return collections;
|
|
937
|
+
}
|
|
938
|
+
function pytestSymbols(script, source) {
|
|
939
|
+
const matches = [...source.matchAll(/^([ \t]*)(?:async\s+)?def\s+(test_[A-Za-z0-9_]+)\s*\([^)]*\)\s*(?:->\s*[^:\r\n]+)?\s*:/gm)];
|
|
940
|
+
const regionStarts = matches.map((match) => pytestFunctionRegionStart(source, match.index));
|
|
941
|
+
const parameterCollections = pytestParameterCollections(source);
|
|
942
|
+
return matches.map((match, index) => {
|
|
943
|
+
const regionStart = regionStarts[index];
|
|
944
|
+
const regionEnd = regionStarts[index + 1] ?? source.length;
|
|
945
|
+
const region = source.slice(match.index, regionEnd);
|
|
946
|
+
const decorators = source.slice(regionStart, match.index);
|
|
947
|
+
const ids = caseIds(region);
|
|
948
|
+
const fromSymbol = symbolCaseId(match[2]);
|
|
949
|
+
if (fromSymbol)
|
|
950
|
+
ids.unshift(fromSymbol);
|
|
951
|
+
const parameterIds = orderedUnique([
|
|
952
|
+
...[...decorators.matchAll(/\bid\s*=\s*["'](TP-[A-Z0-9-]+)["']/g)].map((item) => item[1]),
|
|
953
|
+
...[...parameterCollections.entries()].flatMap(([name, values]) => new RegExp(`\\b${name}\\b`).test(decorators) ? values : []),
|
|
954
|
+
]);
|
|
955
|
+
const assertionTestPoints = metadataTestPoints(region, "Assertion-Test-Points");
|
|
956
|
+
const crossCuttingTestPoints = metadataTestPoints(region, "Cross-Cutting-Test-Points");
|
|
957
|
+
return {
|
|
958
|
+
script,
|
|
959
|
+
symbol: match[2],
|
|
960
|
+
caseIds: orderedUnique(ids),
|
|
961
|
+
testPoints: orderedUnique([...parameterIds, ...assertionTestPoints, ...crossCuttingTestPoints]),
|
|
962
|
+
parameterIds,
|
|
963
|
+
assertionTestPoints,
|
|
964
|
+
crossCuttingTestPoints,
|
|
965
|
+
};
|
|
966
|
+
});
|
|
967
|
+
}
|
|
968
|
+
export async function analyzeBackendTestMarkdownPytestCorrespondence(input) {
|
|
969
|
+
const cases = await parseMarkdownCases(input.workspaceRoot);
|
|
970
|
+
const modulePaths = orderedUnique(cases.map((item) => item.markdownPath));
|
|
971
|
+
const scripts = orderedUnique(cases.flatMap((item) => [...item.declaredScripts, expectedScript(item.markdownPath)]));
|
|
972
|
+
const allSymbols = [];
|
|
973
|
+
const scriptFiles = [];
|
|
974
|
+
for (const script of scripts) {
|
|
975
|
+
const absolute = path.resolve(input.workspaceRoot, script);
|
|
976
|
+
if (!(await exists(absolute)))
|
|
977
|
+
continue;
|
|
978
|
+
scriptFiles.push(absolute);
|
|
979
|
+
allSymbols.push(...pytestSymbols(script, await readFile(absolute, "utf8")));
|
|
980
|
+
}
|
|
981
|
+
const findings = [];
|
|
982
|
+
const entries = [];
|
|
983
|
+
for (const testCase of cases) {
|
|
984
|
+
const expected = expectedScript(testCase.markdownPath);
|
|
985
|
+
const declared = testCase.declaredScripts[0];
|
|
986
|
+
const refs = allSymbols.filter((item) => item.caseIds.includes(testCase.caseId));
|
|
987
|
+
const actualScripts = orderedUnique(refs.map((item) => item.script));
|
|
988
|
+
const symbols = orderedUnique(refs.map((item) => item.symbol));
|
|
989
|
+
const parameterIds = orderedUnique(refs.flatMap((item) => item.parameterIds));
|
|
990
|
+
const assertionBindings = orderedUnique(refs.flatMap((item) => item.assertionTestPoints));
|
|
991
|
+
const crossCuttingBindings = orderedUnique(refs.flatMap((item) => item.crossCuttingTestPoints));
|
|
992
|
+
const variantTestPoints = testCase.testPointBindings.filter((item) => item.mode === "variant").map((item) => item.testPoint);
|
|
993
|
+
const assertionTestPoints = testCase.testPointBindings.filter((item) => item.mode === "assertion").map((item) => item.testPoint);
|
|
994
|
+
const crossCuttingTestPoints = testCase.testPointBindings.filter((item) => item.mode === "cross-cutting").map((item) => item.testPoint);
|
|
995
|
+
const mappedTestPoints = orderedUnique([
|
|
996
|
+
...variantTestPoints.filter((point) => parameterIds.includes(point)),
|
|
997
|
+
...assertionTestPoints.filter((point) => assertionBindings.includes(point)),
|
|
998
|
+
...crossCuttingTestPoints.filter((point) => crossCuttingBindings.includes(point)),
|
|
999
|
+
]);
|
|
1000
|
+
const declaredPrimarySymbol = testCase.declaredPrimarySymbols[0];
|
|
1001
|
+
const cardinality = refs.length === 0 ? "1:0" : refs.length === 1 ? "1:1" : "1:N";
|
|
1002
|
+
let status = refs.length === 0 ? "MISSING_PYTEST" : refs.length > 1 ? "MULTIPLE_PYTEST" : "EXACT_1_TO_1";
|
|
1003
|
+
const entryFindings = [];
|
|
1004
|
+
if (refs.length === 0)
|
|
1005
|
+
entryFindings.push(`${testCase.caseId} has no pytest symbol`);
|
|
1006
|
+
if (refs.length > 1)
|
|
1007
|
+
entryFindings.push(`${testCase.caseId} maps to multiple pytest symbols: ${symbols.join(", ")}`);
|
|
1008
|
+
if (!declared || declared !== expected || actualScripts.some((script) => script !== expected) || !declaredPrimarySymbol || symbols.some((symbol) => symbol !== declaredPrimarySymbol)) {
|
|
1009
|
+
if (refs.length > 0 && status === "EXACT_1_TO_1")
|
|
1010
|
+
status = "SCRIPT_MISMATCH";
|
|
1011
|
+
entryFindings.push(`${testCase.caseId} script/primary-symbol mapping mismatch: declaredScript=${declared ?? "none"}, expectedScript=${expected}, actual=${actualScripts.join(", ") || "none"}, declaredPrimarySymbol=${declaredPrimarySymbol ?? "none"}, actualSymbols=${symbols.join(", ") || "none"}`);
|
|
1012
|
+
}
|
|
1013
|
+
const missingVariant = variantTestPoints.filter((point) => !parameterIds.includes(point));
|
|
1014
|
+
const missingAssertion = assertionTestPoints.filter((point) => !assertionBindings.includes(point));
|
|
1015
|
+
const missingCrossCutting = crossCuttingTestPoints.filter((point) => !crossCuttingBindings.includes(point));
|
|
1016
|
+
const extraPoints = orderedUnique([...parameterIds, ...assertionBindings, ...crossCuttingBindings]).filter((point) => !testCase.testPoints.includes(point));
|
|
1017
|
+
if (testCase.unclassifiedTestPoints.length > 0 || testCase.duplicateBindingTestPoints.length > 0) {
|
|
1018
|
+
if (refs.length > 0 && status === "EXACT_1_TO_1")
|
|
1019
|
+
status = "TEST_POINT_BINDING_DUPLICATE";
|
|
1020
|
+
entryFindings.push(`${testCase.caseId} invalid Markdown Test Point bindings: unclassified=${testCase.unclassifiedTestPoints.join(", ") || "none"}; duplicate=${testCase.duplicateBindingTestPoints.join(", ") || "none"}`);
|
|
1021
|
+
}
|
|
1022
|
+
if (missingVariant.length > 0) {
|
|
1023
|
+
if (refs.length > 0 && status === "EXACT_1_TO_1")
|
|
1024
|
+
status = "VARIANT_PARAMETER_MISSING";
|
|
1025
|
+
entryFindings.push(`${testCase.caseId} missing variant pytest parameter IDs: ${missingVariant.join(", ")}`);
|
|
1026
|
+
}
|
|
1027
|
+
if (missingAssertion.length > 0) {
|
|
1028
|
+
if (refs.length > 0 && status === "EXACT_1_TO_1")
|
|
1029
|
+
status = "ASSERTION_BINDING_MISSING";
|
|
1030
|
+
entryFindings.push(`${testCase.caseId} missing assertion docstring bindings: ${missingAssertion.join(", ")}`);
|
|
1031
|
+
}
|
|
1032
|
+
if (missingCrossCutting.length > 0) {
|
|
1033
|
+
if (refs.length > 0 && status === "EXACT_1_TO_1")
|
|
1034
|
+
status = "CROSS_CUTTING_EVIDENCE_MISSING";
|
|
1035
|
+
entryFindings.push(`${testCase.caseId} missing cross-cutting evidence bindings: ${missingCrossCutting.join(", ")}`);
|
|
1036
|
+
}
|
|
1037
|
+
if (extraPoints.length > 0) {
|
|
1038
|
+
if (refs.length > 0 && status === "EXACT_1_TO_1")
|
|
1039
|
+
status = "TEST_POINT_BINDING_EXTRA";
|
|
1040
|
+
entryFindings.push(`${testCase.caseId} has extra pytest Test Point bindings: ${extraPoints.join(", ")}`);
|
|
1041
|
+
}
|
|
1042
|
+
findings.push(...entryFindings);
|
|
1043
|
+
entries.push({
|
|
1044
|
+
markdownModule: testCase.markdownPath,
|
|
1045
|
+
caseId: testCase.caseId,
|
|
1046
|
+
testPoints: testCase.testPoints,
|
|
1047
|
+
variantTestPoints,
|
|
1048
|
+
assertionTestPoints,
|
|
1049
|
+
crossCuttingTestPoints,
|
|
1050
|
+
mappedTestPoints,
|
|
1051
|
+
...(declared ? { declaredScript: declared } : {}),
|
|
1052
|
+
...(declaredPrimarySymbol ? { declaredPrimarySymbol } : {}),
|
|
1053
|
+
expectedScript: expected,
|
|
1054
|
+
actualScripts,
|
|
1055
|
+
pytestSymbols: symbols,
|
|
1056
|
+
parameterIds,
|
|
1057
|
+
cardinality,
|
|
1058
|
+
status,
|
|
1059
|
+
findings: entryFindings,
|
|
1060
|
+
});
|
|
1061
|
+
}
|
|
1062
|
+
for (const symbol of allSymbols) {
|
|
1063
|
+
const boundKnown = symbol.caseIds.some((id) => cases.some((testCase) => testCase.caseId === id));
|
|
1064
|
+
if (boundKnown)
|
|
1065
|
+
continue;
|
|
1066
|
+
const unknownId = symbol.caseIds[0];
|
|
1067
|
+
const message = unknownId
|
|
1068
|
+
? `${symbol.script}#${symbol.symbol} references unknown Markdown Case ${unknownId}`
|
|
1069
|
+
: `${symbol.script}#${symbol.symbol} has no Markdown Case ID`;
|
|
1070
|
+
findings.push(message);
|
|
1071
|
+
entries.push({
|
|
1072
|
+
markdownModule: "—",
|
|
1073
|
+
...(unknownId ? { caseId: unknownId } : {}),
|
|
1074
|
+
testPoints: symbol.testPoints,
|
|
1075
|
+
variantTestPoints: [],
|
|
1076
|
+
assertionTestPoints: [],
|
|
1077
|
+
crossCuttingTestPoints: [],
|
|
1078
|
+
mappedTestPoints: [],
|
|
1079
|
+
expectedScript: symbol.script,
|
|
1080
|
+
actualScripts: [symbol.script],
|
|
1081
|
+
pytestSymbols: [symbol.symbol],
|
|
1082
|
+
parameterIds: symbol.parameterIds,
|
|
1083
|
+
cardinality: "0:1",
|
|
1084
|
+
status: unknownId ? "EXTRA_PYTEST" : "SYMBOL_MISSING_CASE_ID",
|
|
1085
|
+
findings: [message],
|
|
1086
|
+
});
|
|
1087
|
+
}
|
|
1088
|
+
const exactModuleCount = modulePaths.filter((modulePath) => {
|
|
1089
|
+
const expected = expectedScript(modulePath);
|
|
1090
|
+
const moduleCases = cases.filter((item) => item.markdownPath === modulePath);
|
|
1091
|
+
return moduleCases.every((item) => item.declaredScripts.length === 1 && item.declaredScripts[0] === expected) && scriptFiles.some((file) => path.relative(input.workspaceRoot, file).replaceAll(path.sep, "/") === expected);
|
|
1092
|
+
}).length;
|
|
1093
|
+
const caseEntries = entries.filter((entry) => entry.cardinality !== "0:1");
|
|
1094
|
+
const facts = backendTestMarkdownPytestCorrespondenceFactsSchema.parse({
|
|
1095
|
+
schemaId: "backend-test-markdown-pytest-correspondence-facts-v2",
|
|
1096
|
+
schemaVersion: 2,
|
|
1097
|
+
taskId: input.taskId,
|
|
1098
|
+
status: findings.length === 0 ? "PASS" : "FAIL",
|
|
1099
|
+
inputFiles: await inputFileFacts(input.workspaceRoot, [
|
|
1100
|
+
path.join(input.workspaceRoot, "testcase", "md", "README.md"),
|
|
1101
|
+
...(await markdownModuleFiles(input.workspaceRoot)),
|
|
1102
|
+
...scriptFiles,
|
|
1103
|
+
].filter((file) => file && path.isAbsolute(file))),
|
|
1104
|
+
summary: {
|
|
1105
|
+
markdownModuleCount: modulePaths.length,
|
|
1106
|
+
exactModuleCount,
|
|
1107
|
+
markdownCaseCount: cases.length,
|
|
1108
|
+
exactCorrespondenceCount: caseEntries.filter((entry) => entry.status === "EXACT_1_TO_1").length,
|
|
1109
|
+
primarySymbolCount: orderedUnique(caseEntries.flatMap((entry) => entry.pytestSymbols)).length,
|
|
1110
|
+
missingPytestCount: caseEntries.filter((entry) => entry.status === "MISSING_PYTEST").length,
|
|
1111
|
+
multiplePytestCount: caseEntries.filter((entry) => entry.status === "MULTIPLE_PYTEST").length,
|
|
1112
|
+
extraPytestCount: entries.filter((entry) => entry.cardinality === "0:1").length,
|
|
1113
|
+
scriptMismatchCount: caseEntries.filter((entry) => entry.status === "SCRIPT_MISMATCH").length,
|
|
1114
|
+
testPointCount: cases.reduce((sum, item) => sum + item.testPoints.length, 0),
|
|
1115
|
+
mappedTestPointCount: caseEntries.reduce((sum, entry) => sum + entry.mappedTestPoints.length, 0),
|
|
1116
|
+
variantTestPointCount: cases.reduce((sum, item) => sum + item.testPointBindings.filter((binding) => binding.mode === "variant").length, 0),
|
|
1117
|
+
assertionTestPointCount: cases.reduce((sum, item) => sum + item.testPointBindings.filter((binding) => binding.mode === "assertion").length, 0),
|
|
1118
|
+
crossCuttingTestPointCount: cases.reduce((sum, item) => sum + item.testPointBindings.filter((binding) => binding.mode === "cross-cutting").length, 0),
|
|
1119
|
+
unclassifiedTestPointCount: cases.reduce((sum, item) => sum + item.unclassifiedTestPoints.length, 0),
|
|
1120
|
+
duplicateBindingTestPointCount: cases.reduce((sum, item) => sum + item.duplicateBindingTestPoints.length, 0),
|
|
1121
|
+
},
|
|
1122
|
+
entries,
|
|
1123
|
+
findings: orderedUnique(findings),
|
|
1124
|
+
});
|
|
1125
|
+
const markdown = [
|
|
1126
|
+
"# Backend Test Markdown → pytest Correspondence",
|
|
1127
|
+
"",
|
|
1128
|
+
"## Status",
|
|
1129
|
+
"",
|
|
1130
|
+
facts.status,
|
|
1131
|
+
"",
|
|
1132
|
+
"## Summary",
|
|
1133
|
+
"",
|
|
1134
|
+
`- Markdown modules: ${facts.summary.markdownModuleCount}`,
|
|
1135
|
+
`- Exact module mappings: ${facts.summary.exactModuleCount}`,
|
|
1136
|
+
`- Markdown Cases: ${facts.summary.markdownCaseCount}`,
|
|
1137
|
+
`- Exact 1:1: ${facts.summary.exactCorrespondenceCount}`,
|
|
1138
|
+
`- Primary pytest symbols: ${facts.summary.primarySymbolCount}`,
|
|
1139
|
+
`- Missing pytest: ${facts.summary.missingPytestCount}`,
|
|
1140
|
+
`- Multiple pytest: ${facts.summary.multiplePytestCount}`,
|
|
1141
|
+
`- Extra pytest: ${facts.summary.extraPytestCount}`,
|
|
1142
|
+
`- Test Points: ${facts.summary.mappedTestPointCount}/${facts.summary.testPointCount}`,
|
|
1143
|
+
`- Variant Test Points: ${facts.summary.variantTestPointCount}`,
|
|
1144
|
+
`- Assertion Test Points: ${facts.summary.assertionTestPointCount}`,
|
|
1145
|
+
`- Cross-Cutting Test Points: ${facts.summary.crossCuttingTestPointCount}`,
|
|
1146
|
+
`- Unclassified Test Points: ${facts.summary.unclassifiedTestPointCount}`,
|
|
1147
|
+
`- Duplicate Test Point Bindings: ${facts.summary.duplicateBindingTestPointCount}`,
|
|
1148
|
+
"",
|
|
1149
|
+
"## Correspondence Matrix",
|
|
1150
|
+
"",
|
|
1151
|
+
"| Markdown Module | Case ID | Variant | Assertion | Cross-Cutting | Mapped | Declared Script | Primary Symbol | Actual Script | Pytest Symbol | Parameter IDs | Cardinality | Status |",
|
|
1152
|
+
"|---|---|---|---|---|---|---|---|---|---|---|---|---|",
|
|
1153
|
+
...facts.entries.map((entry) => `| ${entry.markdownModule} | ${entry.caseId ?? "—"} | ${entry.variantTestPoints.join(", ") || "—"} | ${entry.assertionTestPoints.join(", ") || "—"} | ${entry.crossCuttingTestPoints.join(", ") || "—"} | ${entry.mappedTestPoints.join(", ") || "—"} | ${entry.declaredScript ?? "—"} | ${entry.declaredPrimarySymbol ?? "—"} | ${entry.actualScripts.join(", ") || "—"} | ${entry.pytestSymbols.join(", ") || "—"} | ${entry.parameterIds.join(", ") || "—"} | ${entry.cardinality} | ${entry.status} |`),
|
|
1154
|
+
"",
|
|
1155
|
+
"## Findings",
|
|
1156
|
+
"",
|
|
1157
|
+
...(facts.findings.length ? facts.findings.map((item) => `- ${item}`) : ["- None"]),
|
|
1158
|
+
"",
|
|
1159
|
+
].join("\n");
|
|
1160
|
+
return { facts, markdown };
|
|
1161
|
+
}
|
|
1162
|
+
async function readFacts(filePath, schema) {
|
|
1163
|
+
try {
|
|
1164
|
+
const parsed = schema.safeParse(JSON.parse(await readFile(filePath, "utf8")));
|
|
1165
|
+
if (!parsed.success)
|
|
1166
|
+
return { issue: `${path.basename(filePath)} schema invalid: ${parsed.error.issues[0]?.message ?? "unknown"}` };
|
|
1167
|
+
return { value: parsed.data };
|
|
1168
|
+
}
|
|
1169
|
+
catch (error) {
|
|
1170
|
+
return { issue: `${path.basename(filePath)} unavailable: ${error instanceof Error ? error.message : String(error)}` };
|
|
1171
|
+
}
|
|
1172
|
+
}
|
|
1173
|
+
async function freshnessIssues(workspaceRoot, inputFiles) {
|
|
1174
|
+
if (!workspaceRoot)
|
|
1175
|
+
return [];
|
|
1176
|
+
const issues = [];
|
|
1177
|
+
for (const item of inputFiles) {
|
|
1178
|
+
const absolute = path.resolve(workspaceRoot, item.path);
|
|
1179
|
+
if (!(await exists(absolute)))
|
|
1180
|
+
issues.push(`input file missing since facts generation: ${item.path}`);
|
|
1181
|
+
else if ((await fileHash(absolute)) !== item.sha256)
|
|
1182
|
+
issues.push(`input file hash changed since facts generation: ${item.path}`);
|
|
1183
|
+
}
|
|
1184
|
+
return issues;
|
|
1185
|
+
}
|
|
1186
|
+
export async function materializeBackendTestCaseManifestFromFacts(input) {
|
|
1187
|
+
const contractsDir = path.join(input.runDir, "contracts");
|
|
1188
|
+
await mkdir(contractsDir, { recursive: true });
|
|
1189
|
+
const coveragePath = path.join(contractsDir, "backend-test-case-coverage-facts.json");
|
|
1190
|
+
const correspondencePath = path.join(contractsDir, "backend-test-markdown-pytest-correspondence-facts.json");
|
|
1191
|
+
const coverageResult = await readFacts(coveragePath, backendTestCaseCoverageFactsSchema);
|
|
1192
|
+
const correspondenceResult = await readFacts(correspondencePath, backendTestMarkdownPytestCorrespondenceFactsSchema);
|
|
1193
|
+
const coverageCases = new Map((coverageResult.value?.cases ?? []).map((item) => [item.caseId, item]));
|
|
1194
|
+
const correspondenceCases = new Map((correspondenceResult.value?.entries ?? []).filter((item) => item.caseId && item.cardinality !== "0:1").map((item) => [item.caseId, item]));
|
|
1195
|
+
const keyIssues = coverageResult.value && correspondenceResult.value ? [
|
|
1196
|
+
...[...coverageCases.keys()].filter((caseId) => !correspondenceCases.has(caseId)).map((caseId) => `coverage Case missing from correspondence facts: ${caseId}`),
|
|
1197
|
+
...[...correspondenceCases.keys()].filter((caseId) => !coverageCases.has(caseId)).map((caseId) => `correspondence Case missing from coverage facts: ${caseId}`),
|
|
1198
|
+
...[...coverageCases.entries()].flatMap(([caseId, testCase]) => {
|
|
1199
|
+
const mapping = correspondenceCases.get(caseId);
|
|
1200
|
+
return mapping && JSON.stringify([...testCase.testPoints].sort()) !== JSON.stringify([...mapping.testPoints].sort())
|
|
1201
|
+
? [`Case Test Point keys disagree between facts: ${caseId}`]
|
|
1202
|
+
: [];
|
|
1203
|
+
}),
|
|
1204
|
+
] : [];
|
|
1205
|
+
const sourceFactsIssues = orderedUnique([
|
|
1206
|
+
...(coverageResult.issue ? [coverageResult.issue] : []),
|
|
1207
|
+
...(correspondenceResult.issue ? [correspondenceResult.issue] : []),
|
|
1208
|
+
...(coverageResult.value?.taskId !== undefined && coverageResult.value.taskId !== input.sourceBinding.taskId ? ["coverage facts taskId does not match source binding"] : []),
|
|
1209
|
+
...(coverageResult.value && JSON.stringify(coverageResult.value.sourceBinding) !== JSON.stringify(input.sourceBinding) ? ["coverage facts source binding does not match manifest source binding"] : []),
|
|
1210
|
+
...(correspondenceResult.value?.taskId !== undefined && correspondenceResult.value.taskId !== input.sourceBinding.taskId ? ["correspondence facts taskId does not match source binding"] : []),
|
|
1211
|
+
...keyIssues,
|
|
1212
|
+
...(coverageResult.value ? await freshnessIssues(input.workspaceRoot, coverageResult.value.inputFiles) : []),
|
|
1213
|
+
...(correspondenceResult.value ? await freshnessIssues(input.workspaceRoot, correspondenceResult.value.inputFiles) : []),
|
|
1214
|
+
]);
|
|
1215
|
+
const materializationStatus = !coverageResult.value && !correspondenceResult.value
|
|
1216
|
+
? "unavailable"
|
|
1217
|
+
: sourceFactsIssues.length > 0 || !coverageResult.value || !correspondenceResult.value
|
|
1218
|
+
? "partial"
|
|
1219
|
+
: "available";
|
|
1220
|
+
const correspondenceByCase = new Map((correspondenceResult.value?.entries ?? [])
|
|
1221
|
+
.filter((entry) => entry.caseId && entry.cardinality !== "0:1")
|
|
1222
|
+
.map((entry) => [entry.caseId, entry]));
|
|
1223
|
+
const manifestCases = (coverageResult.value?.cases ?? []).map((testCase) => {
|
|
1224
|
+
const mapping = correspondenceByCase.get(testCase.caseId);
|
|
1225
|
+
const generated = Boolean(mapping && mapping.pytestSymbols.length > 0 && mapping.actualScripts.length > 0);
|
|
1226
|
+
return {
|
|
1227
|
+
caseId: testCase.caseId,
|
|
1228
|
+
acIds: testCase.acIds,
|
|
1229
|
+
title: testCase.title,
|
|
1230
|
+
category: inferCategory(testCase.scenarioTypes),
|
|
1231
|
+
automationStatus: generated ? "generated" : "planned",
|
|
1232
|
+
ruleRefs: testCase.ruleKeys,
|
|
1233
|
+
...(generated ? { file: mapping.actualScripts[0], symbol: mapping.pytestSymbols[0] } : { gapReason: "pytest correspondence is missing or unavailable" }),
|
|
1234
|
+
};
|
|
1235
|
+
});
|
|
1236
|
+
const base = {
|
|
1237
|
+
schemaVersion: 1,
|
|
1238
|
+
sourceBinding: input.sourceBinding,
|
|
1239
|
+
cases: manifestCases,
|
|
1240
|
+
evidenceGaps: [],
|
|
1241
|
+
};
|
|
1242
|
+
const coverageSummary = materializationStatus === "available" ? computeCaseManifestCoverageSummary(base) : undefined;
|
|
1243
|
+
const manifest = backendTestCaseManifestSchema.parse({
|
|
1244
|
+
...base,
|
|
1245
|
+
materializationStatus,
|
|
1246
|
+
sourceFactsIssues,
|
|
1247
|
+
coverageScope: coverageResult.value?.coverageScope,
|
|
1248
|
+
ruleCoverageSummary: coverageResult.value?.summary,
|
|
1249
|
+
correspondenceSummary: correspondenceResult.value?.summary,
|
|
1250
|
+
artifactRefs: {
|
|
1251
|
+
caseCoverageFacts: await artifactRef(input.runDir, coveragePath),
|
|
1252
|
+
correspondenceFacts: await artifactRef(input.runDir, correspondencePath),
|
|
1253
|
+
},
|
|
1254
|
+
...(coverageSummary ? { coverageSummary } : {}),
|
|
1255
|
+
});
|
|
1256
|
+
await writeFile(path.join(contractsDir, "backend-test-case-manifest.json"), JSON.stringify(manifest, null, 2), "utf8");
|
|
1257
|
+
return manifest;
|
|
1258
|
+
}
|
|
1259
|
+
function inferCategory(types) {
|
|
1260
|
+
const text = types.join(" ").toLowerCase();
|
|
1261
|
+
if (/negative|invalid|error/.test(text))
|
|
1262
|
+
return "negative";
|
|
1263
|
+
if (/boundary|format|enum/.test(text))
|
|
1264
|
+
return "boundary";
|
|
1265
|
+
if (/state|business|lifecycle|uniqueness/.test(text))
|
|
1266
|
+
return "state-transition";
|
|
1267
|
+
if (/auth|permission/.test(text))
|
|
1268
|
+
return "auth";
|
|
1269
|
+
if (/timeout/.test(text))
|
|
1270
|
+
return "timeout";
|
|
1271
|
+
if (/concurr/.test(text))
|
|
1272
|
+
return "concurrency";
|
|
1273
|
+
if (/positive|requirement/.test(text))
|
|
1274
|
+
return "positive";
|
|
1275
|
+
return "other";
|
|
1276
|
+
}
|
|
1277
|
+
async function artifactRef(runDir, absolutePath) {
|
|
1278
|
+
if (!(await exists(absolutePath)))
|
|
1279
|
+
return { path: path.relative(runDir, absolutePath).replaceAll(path.sep, "/") };
|
|
1280
|
+
return { path: path.relative(runDir, absolutePath).replaceAll(path.sep, "/"), sha256: await fileHash(absolutePath) };
|
|
1281
|
+
}
|