@guanlin4924/devflow-pi 0.1.19 → 0.1.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -0
- package/package.json +3 -3
- package/plugins/devflow-core/extensions/devflow-core.ts +152 -3
- package/plugins/devflow-core/skills/workflow-orchestrator/scripts/acceptance_test_runner.py +394 -0
- package/plugins/devflow-core/skills/workflow-orchestrator/scripts/acceptance_test_suite.rb +86 -0
- package/plugins/devflow-core/skills/workflow-orchestrator/scripts/advance_state.rb +192 -1
- package/plugins/devflow-core/skills/workflow-orchestrator/scripts/api_test_matrix.py +829 -0
- package/plugins/devflow-core/skills/workflow-orchestrator/scripts/execution_contract.rb +7 -5
- package/plugins/devflow-core/skills/workflow-orchestrator/scripts/init_state.rb +11 -1
- package/plugins/devflow-core/skills/workflow-orchestrator/scripts/inspect_state.rb +5 -1
- package/plugins/devflow-core/skills/workflow-orchestrator/scripts/machine_evidence.rb +39 -3
- package/plugins/devflow-core/skills/workflow-orchestrator/scripts/requirement_graph.rb +309 -23
- package/plugins/devflow-core/skills/workflow-orchestrator/scripts/tdd_evidence.py +311 -0
- package/plugins/devflow-core/skills/workflow-orchestrator/scripts/test_command_policy.py +53 -0
- package/plugins/devflow-core/skills/workflow-orchestrator/scripts/test_task_binding_check.py +254 -0
- package/plugins/devflow-core/skills/workflow-orchestrator/templates/feature-development.yaml +109 -4
- package/plugins/devflow-core/skills/workflow-orchestrator/templates/uat-remediation.yaml +21 -1
- package/plugins/devflow-core/skills/workflow-project-context/SKILL.md +17 -0
- package/plugins/devflow-design/skills/workflow-api-contract/SKILL.md +38 -0
- package/plugins/devflow-implementation/agents/implementation-worker.md +3 -0
- package/plugins/devflow-implementation/skills/workflow-change-plan/SKILL.md +7 -0
- package/plugins/devflow-implementation/skills/workflow-change-plan/references/output-template.md +2 -0
- package/plugins/devflow-implementation/skills/workflow-execute-phase/SKILL.md +3 -3
- package/plugins/devflow-quality/skills/workflow-acceptance-test-design/SKILL.md +63 -0
- package/plugins/devflow-quality/skills/workflow-acceptance-test-design/agents/openai.yaml +4 -0
- package/plugins/devflow-quality/skills/workflow-generate-mainflow-cases/SKILL.md +7 -6
- package/plugins/devflow-quality/skills/workflow-generate-mainflow-cases/references/intermediate-format.md +3 -0
- package/plugins/devflow-quality/skills/workflow-generate-mainflow-cases/scripts/compile_dsl.py +49 -3
- package/plugins/devflow-quality/skills/workflow-run-acceptance-tests/SKILL.md +48 -0
- package/plugins/devflow-quality/skills/workflow-run-acceptance-tests/agents/openai.yaml +4 -0
- package/plugins/devflow-quality/skills/workflow-verify-work/SKILL.md +9 -3
- package/src/constants.ts +1 -1
- package/src/doctor.ts +6 -2
- package/src/project-context.ts +11 -1
- package/src/runtime.ts +7 -0
- package/src/usage-stats.ts +3 -3
|
@@ -0,0 +1,829 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Compile a testable API contract into complete, portable acceptance cases.
|
|
3
|
+
|
|
4
|
+
This is deliberately a deterministic Core script, not a prompt convention. A
|
|
5
|
+
plain Markdown request shape cannot prove a maximum length, a minimum value or
|
|
6
|
+
an error-code assertion. API nodes therefore carry a narrow ``test_schema``
|
|
7
|
+
which supplies the facts needed to derive those cases. The compiler creates:
|
|
8
|
+
|
|
9
|
+
* one valid pairwise set (valid values only),
|
|
10
|
+
* one negative case for every required/null/type/range/format/enum partition,
|
|
11
|
+
* one scenario case for every remaining declared error code, and
|
|
12
|
+
* portable HTTP execution definitions plus a binding template.
|
|
13
|
+
|
|
14
|
+
Invalid values are intentionally never mixed into pairwise rows: two invalid
|
|
15
|
+
inputs make the expected error ambiguous and hide a server accepting one of
|
|
16
|
+
them. They are tested one defect at a time and must map to exactly one error
|
|
17
|
+
catalogue entry.
|
|
18
|
+
"""
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import argparse
|
|
22
|
+
import copy
|
|
23
|
+
import hashlib
|
|
24
|
+
import json
|
|
25
|
+
import re
|
|
26
|
+
from itertools import product
|
|
27
|
+
from pathlib import Path
|
|
28
|
+
from typing import Any, Iterable
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
FEATURE_RE = re.compile(r"^[^/\\\x00-\x1f\x7f]+$")
|
|
32
|
+
TYPES = {"string", "integer", "number", "boolean", "array", "object"}
|
|
33
|
+
LOCATIONS = {"body", "query", "path", "headers"}
|
|
34
|
+
MISSING = object()
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def canonical(value: Any) -> str:
|
|
38
|
+
return json.dumps(value, ensure_ascii=False, separators=(",", ":"))
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def digest(value: Any) -> str:
|
|
42
|
+
return hashlib.sha256(canonical(value).encode("utf-8")).hexdigest()
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def read_json(path: Path, errors: list[str], label: str) -> dict[str, Any] | None:
|
|
46
|
+
try:
|
|
47
|
+
value = json.loads(path.read_text(encoding="utf-8"))
|
|
48
|
+
except FileNotFoundError:
|
|
49
|
+
errors.append(f"missing {label}: {path.as_posix()}")
|
|
50
|
+
return None
|
|
51
|
+
except json.JSONDecodeError as error:
|
|
52
|
+
errors.append(f"invalid {label}: {error.msg}")
|
|
53
|
+
return None
|
|
54
|
+
if not isinstance(value, dict):
|
|
55
|
+
errors.append(f"invalid {label}: root must be an object")
|
|
56
|
+
return None
|
|
57
|
+
return value
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def write_json(path: Path, value: dict[str, Any]) -> None:
|
|
61
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
62
|
+
temporary = path.with_name(f".{path.name}.tmp")
|
|
63
|
+
temporary.write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
64
|
+
temporary.replace(path)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def write_text(path: Path, value: str) -> None:
|
|
68
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
69
|
+
temporary = path.with_name(f".{path.name}.tmp")
|
|
70
|
+
temporary.write_text(value, encoding="utf-8")
|
|
71
|
+
temporary.replace(path)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def paths(root: Path, feature: str) -> tuple[Path, Path, Path, Path]:
|
|
75
|
+
requirements = root / ".planning" / "requirements" / feature
|
|
76
|
+
feature_dir = root / ".planning" / "features" / feature
|
|
77
|
+
return (
|
|
78
|
+
requirements / "governance.json",
|
|
79
|
+
requirements / "api-test-matrix.json",
|
|
80
|
+
feature_dir / "完整API验收测试用例.md",
|
|
81
|
+
feature_dir / "api-acceptance-bindings.template.json",
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def json_pointer_tokens(pointer: str) -> list[str]:
|
|
86
|
+
if pointer == "":
|
|
87
|
+
return []
|
|
88
|
+
if not isinstance(pointer, str) or not pointer.startswith("/"):
|
|
89
|
+
raise ValueError("pointer must be an RFC 6901 JSON pointer")
|
|
90
|
+
return [part.replace("~1", "/").replace("~0", "~") for part in pointer[1:].split("/")]
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def pointer_get(document: Any, pointer: str) -> Any:
|
|
94
|
+
current = document
|
|
95
|
+
for token in json_pointer_tokens(pointer):
|
|
96
|
+
if isinstance(current, list):
|
|
97
|
+
if not token.isdigit() or int(token) >= len(current):
|
|
98
|
+
raise KeyError(pointer)
|
|
99
|
+
current = current[int(token)]
|
|
100
|
+
elif isinstance(current, dict):
|
|
101
|
+
if token not in current:
|
|
102
|
+
raise KeyError(pointer)
|
|
103
|
+
current = current[token]
|
|
104
|
+
else:
|
|
105
|
+
raise KeyError(pointer)
|
|
106
|
+
return current
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def pointer_set(document: Any, pointer: str, value: Any) -> None:
|
|
110
|
+
tokens = json_pointer_tokens(pointer)
|
|
111
|
+
if not tokens:
|
|
112
|
+
raise ValueError("root replacement is not supported")
|
|
113
|
+
current = document
|
|
114
|
+
for token in tokens[:-1]:
|
|
115
|
+
if isinstance(current, list):
|
|
116
|
+
if not token.isdigit() or int(token) >= len(current):
|
|
117
|
+
raise KeyError(pointer)
|
|
118
|
+
current = current[int(token)]
|
|
119
|
+
elif isinstance(current, dict):
|
|
120
|
+
if token not in current:
|
|
121
|
+
raise KeyError(pointer)
|
|
122
|
+
current = current[token]
|
|
123
|
+
else:
|
|
124
|
+
raise KeyError(pointer)
|
|
125
|
+
tail = tokens[-1]
|
|
126
|
+
if isinstance(current, list):
|
|
127
|
+
if not tail.isdigit() or int(tail) >= len(current):
|
|
128
|
+
raise KeyError(pointer)
|
|
129
|
+
current[int(tail)] = value
|
|
130
|
+
elif isinstance(current, dict):
|
|
131
|
+
current[tail] = value
|
|
132
|
+
else:
|
|
133
|
+
raise KeyError(pointer)
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def pointer_remove(document: Any, pointer: str) -> None:
|
|
137
|
+
tokens = json_pointer_tokens(pointer)
|
|
138
|
+
if not tokens:
|
|
139
|
+
raise ValueError("root removal is not supported")
|
|
140
|
+
current = document
|
|
141
|
+
for token in tokens[:-1]:
|
|
142
|
+
if isinstance(current, list):
|
|
143
|
+
if not token.isdigit() or int(token) >= len(current):
|
|
144
|
+
raise KeyError(pointer)
|
|
145
|
+
current = current[int(token)]
|
|
146
|
+
elif isinstance(current, dict):
|
|
147
|
+
if token not in current:
|
|
148
|
+
raise KeyError(pointer)
|
|
149
|
+
current = current[token]
|
|
150
|
+
else:
|
|
151
|
+
raise KeyError(pointer)
|
|
152
|
+
tail = tokens[-1]
|
|
153
|
+
if isinstance(current, list):
|
|
154
|
+
if not tail.isdigit() or int(tail) >= len(current):
|
|
155
|
+
raise KeyError(pointer)
|
|
156
|
+
current.pop(int(tail))
|
|
157
|
+
elif isinstance(current, dict):
|
|
158
|
+
if tail not in current:
|
|
159
|
+
raise KeyError(pointer)
|
|
160
|
+
del current[tail]
|
|
161
|
+
else:
|
|
162
|
+
raise KeyError(pointer)
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def unique(values: Iterable[Any]) -> list[Any]:
|
|
166
|
+
result: list[Any] = []
|
|
167
|
+
seen: set[str] = set()
|
|
168
|
+
for value in values:
|
|
169
|
+
key = "<omit>" if value is MISSING else canonical(value)
|
|
170
|
+
if key not in seen:
|
|
171
|
+
seen.add(key)
|
|
172
|
+
result.append(value)
|
|
173
|
+
return result
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def stable_value(value: Any) -> str:
|
|
177
|
+
return "<omit>" if value is MISSING else canonical(value)
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def api_nodes(graph: dict[str, Any]) -> list[dict[str, Any]]:
|
|
181
|
+
return [node for node in graph.get("nodes", []) if isinstance(node, dict) and node.get("kind") == "api"]
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def api_fingerprint(graph: dict[str, Any]) -> str:
|
|
185
|
+
source = [
|
|
186
|
+
{"id": node.get("id"), "data": node.get("data")}
|
|
187
|
+
for node in api_nodes(graph)
|
|
188
|
+
]
|
|
189
|
+
return hashlib.sha256(json.dumps(source, ensure_ascii=False, separators=(",", ":"), sort_keys=True).encode("utf-8")).hexdigest()
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def obligation_id(api_id: str, field_id: str, partition: str) -> str:
|
|
193
|
+
return f"OBL-{api_id}-{field_id}-{partition}".upper().replace("_", "-")
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def case_key(api_id: str, category: str, sequence: int) -> str:
|
|
197
|
+
return f"ATC-{api_id}-{category}-{sequence:04d}".upper().replace("_", "-")
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def error_code_error(api_id: str, field_id: str, partition: str) -> str:
|
|
201
|
+
return f"{api_id}: field {field_id} partition {partition} has no unique error mapping"
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def standard_invalid_format(name: str) -> Any | None:
|
|
205
|
+
return {
|
|
206
|
+
"uuid": "not-a-uuid",
|
|
207
|
+
"email": "not-an-email",
|
|
208
|
+
"date": "2026-99-99",
|
|
209
|
+
"date-time": "not-a-datetime",
|
|
210
|
+
"ipv4": "999.999.999.999",
|
|
211
|
+
"ipv6": "not-an-ip",
|
|
212
|
+
"uri": "not a uri",
|
|
213
|
+
"url": "not a url",
|
|
214
|
+
}.get(name)
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def wrong_type(value_type: str) -> Any:
|
|
218
|
+
return {
|
|
219
|
+
"string": {"unexpected": "object"},
|
|
220
|
+
"integer": "not-an-integer",
|
|
221
|
+
"number": "not-a-number",
|
|
222
|
+
"boolean": "not-a-boolean",
|
|
223
|
+
"array": "not-an-array",
|
|
224
|
+
"object": "not-an-object",
|
|
225
|
+
}[value_type]
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def boundary_keys(value_type: str) -> list[str]:
|
|
229
|
+
if value_type == "string":
|
|
230
|
+
return ["min_length", "max_length", "format", "pattern", "enum"]
|
|
231
|
+
if value_type in {"integer", "number"}:
|
|
232
|
+
return ["minimum", "maximum", "enum"]
|
|
233
|
+
if value_type == "array":
|
|
234
|
+
return ["min_items", "max_items"]
|
|
235
|
+
return []
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def coverage_declared(field: dict[str, Any], key: str) -> bool:
|
|
239
|
+
constraints = field.get("constraints") if isinstance(field.get("constraints"), dict) else {}
|
|
240
|
+
if key in constraints:
|
|
241
|
+
return True
|
|
242
|
+
coverage = field.get("coverage") if isinstance(field.get("coverage"), dict) else {}
|
|
243
|
+
item = coverage.get(key)
|
|
244
|
+
return isinstance(item, dict) and item.get("state") == "not_applicable" and isinstance(item.get("reason"), str) and bool(item["reason"].strip())
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
def validate_field(api_id: str, schema: dict[str, Any], field: Any, seen: set[str], errors: list[str]) -> None:
|
|
248
|
+
if not isinstance(field, dict):
|
|
249
|
+
errors.append(f"{api_id}: test_schema.fields entries must be objects")
|
|
250
|
+
return
|
|
251
|
+
field_id = field.get("field_key")
|
|
252
|
+
if not isinstance(field_id, str) or not field_id.strip():
|
|
253
|
+
errors.append(f"{api_id}: test_schema field_key is required")
|
|
254
|
+
return
|
|
255
|
+
if field_id in seen:
|
|
256
|
+
errors.append(f"{api_id}: duplicate test_schema field id {field_id}")
|
|
257
|
+
seen.add(field_id)
|
|
258
|
+
value_type = field.get("type")
|
|
259
|
+
if value_type not in TYPES:
|
|
260
|
+
errors.append(f"{api_id}: field {field_id} type is invalid")
|
|
261
|
+
if field.get("location") not in LOCATIONS:
|
|
262
|
+
errors.append(f"{api_id}: field {field_id} location is invalid")
|
|
263
|
+
pointer = field.get("pointer")
|
|
264
|
+
base_value: Any = MISSING
|
|
265
|
+
try:
|
|
266
|
+
json_pointer_tokens(pointer)
|
|
267
|
+
request = schema.get("valid_request") if isinstance(schema.get("valid_request"), dict) else {}
|
|
268
|
+
location_value = request.get(field.get("location")) if isinstance(field.get("location"), str) else None
|
|
269
|
+
base_value = pointer_get(location_value, pointer)
|
|
270
|
+
except (KeyError, TypeError, ValueError):
|
|
271
|
+
errors.append(f"{api_id}: field {field_id} pointer must resolve in valid_request.{field.get('location')}")
|
|
272
|
+
if not isinstance(field.get("required"), bool):
|
|
273
|
+
errors.append(f"{api_id}: field {field_id} required must be boolean")
|
|
274
|
+
if not isinstance(field.get("nullable"), bool):
|
|
275
|
+
errors.append(f"{api_id}: field {field_id} nullable must be boolean")
|
|
276
|
+
constraints = field.get("constraints", {})
|
|
277
|
+
if not isinstance(constraints, dict):
|
|
278
|
+
errors.append(f"{api_id}: field {field_id} constraints must be an object")
|
|
279
|
+
constraints = {}
|
|
280
|
+
for key in boundary_keys(value_type if isinstance(value_type, str) else ""):
|
|
281
|
+
if not coverage_declared(field, key):
|
|
282
|
+
errors.append(f"{api_id}: field {field_id} must declare {key} or coverage.{key}=not_applicable with a reason")
|
|
283
|
+
for key in ("min_length", "max_length", "min_items", "max_items"):
|
|
284
|
+
if key in constraints and (not isinstance(constraints[key], int) or constraints[key] < 0):
|
|
285
|
+
errors.append(f"{api_id}: field {field_id} constraint {key} must be a non-negative integer")
|
|
286
|
+
for key in ("minimum", "maximum"):
|
|
287
|
+
if key in constraints and (not isinstance(constraints[key], (int, float)) or isinstance(constraints[key], bool)):
|
|
288
|
+
errors.append(f"{api_id}: field {field_id} constraint {key} must be a number")
|
|
289
|
+
if isinstance(constraints.get("minimum"), (int, float)) and isinstance(constraints.get("maximum"), (int, float)) and constraints["minimum"] > constraints["maximum"]:
|
|
290
|
+
errors.append(f"{api_id}: field {field_id} minimum cannot exceed maximum")
|
|
291
|
+
if isinstance(constraints.get("min_length"), int) and isinstance(constraints.get("max_length"), int) and constraints["min_length"] > constraints["max_length"]:
|
|
292
|
+
errors.append(f"{api_id}: field {field_id} min_length cannot exceed max_length")
|
|
293
|
+
if isinstance(constraints.get("min_items"), int) and isinstance(constraints.get("max_items"), int) and constraints["min_items"] > constraints["max_items"]:
|
|
294
|
+
errors.append(f"{api_id}: field {field_id} min_items cannot exceed max_items")
|
|
295
|
+
if "enum" in constraints and (not isinstance(constraints["enum"], list) or not constraints["enum"]):
|
|
296
|
+
errors.append(f"{api_id}: field {field_id} enum must be a non-empty array")
|
|
297
|
+
if "pattern" in constraints:
|
|
298
|
+
examples = field.get("examples") if isinstance(field.get("examples"), dict) else {}
|
|
299
|
+
if not isinstance(examples.get("invalid_pattern"), (str, int, float, bool, list, dict)) and examples.get("invalid_pattern") is not None:
|
|
300
|
+
errors.append(f"{api_id}: field {field_id} examples.invalid_pattern must be JSON data")
|
|
301
|
+
if "invalid_pattern" not in examples:
|
|
302
|
+
errors.append(f"{api_id}: field {field_id} pattern requires examples.invalid_pattern")
|
|
303
|
+
if "enum" in constraints:
|
|
304
|
+
examples = field.get("examples") if isinstance(field.get("examples"), dict) else {}
|
|
305
|
+
if "invalid_enum" not in examples:
|
|
306
|
+
errors.append(f"{api_id}: field {field_id} enum requires examples.invalid_enum")
|
|
307
|
+
# A generic "xxxx" can accidentally violate a format/pattern/enum at the
|
|
308
|
+
# same time as a length/range boundary. In such intersecting constraints
|
|
309
|
+
# require the contract author to supply an isolating value; this is the
|
|
310
|
+
# only honest alternative to asserting an arbitrary error code.
|
|
311
|
+
examples = field.get("examples") if isinstance(field.get("examples"), dict) else {}
|
|
312
|
+
intersecting = any(key in constraints for key in ("format", "pattern", "enum"))
|
|
313
|
+
if intersecting and value_type == "string":
|
|
314
|
+
for constraint, example in (("min_length", "at_min_length"), ("min_length", "below_min_length"), ("max_length", "at_max_length"), ("max_length", "above_max_length")):
|
|
315
|
+
if constraint in constraints and example not in examples:
|
|
316
|
+
errors.append(f"{api_id}: field {field_id} {constraint} with format/pattern/enum requires examples.{example} to isolate the boundary")
|
|
317
|
+
if "enum" in constraints and value_type in {"integer", "number"}:
|
|
318
|
+
for constraint, example in (("minimum", "at_minimum"), ("minimum", "below_minimum"), ("maximum", "at_maximum"), ("maximum", "above_maximum")):
|
|
319
|
+
if constraint in constraints and example not in examples:
|
|
320
|
+
errors.append(f"{api_id}: field {field_id} {constraint} with enum requires examples.{example} to isolate the boundary")
|
|
321
|
+
if value_type == "string":
|
|
322
|
+
checks = (("min_length", "at_min_length", lambda value, limit: len(value) == limit), ("min_length", "below_min_length", lambda value, limit: len(value) < limit), ("max_length", "at_max_length", lambda value, limit: len(value) == limit), ("max_length", "above_max_length", lambda value, limit: len(value) > limit))
|
|
323
|
+
for constraint, example, predicate in checks:
|
|
324
|
+
if constraint in constraints and example in examples and (not isinstance(examples[example], str) or not predicate(examples[example], constraints[constraint])):
|
|
325
|
+
errors.append(f"{api_id}: field {field_id} examples.{example} does not satisfy its declared {constraint} boundary")
|
|
326
|
+
if value_type in {"integer", "number"}:
|
|
327
|
+
checks = (("minimum", "at_minimum", lambda value, limit: value == limit), ("minimum", "below_minimum", lambda value, limit: value < limit), ("maximum", "at_maximum", lambda value, limit: value == limit), ("maximum", "above_maximum", lambda value, limit: value > limit))
|
|
328
|
+
for constraint, example, predicate in checks:
|
|
329
|
+
if constraint in constraints and example in examples and (not isinstance(examples[example], (int, float)) or isinstance(examples[example], bool) or not predicate(examples[example], constraints[constraint])):
|
|
330
|
+
errors.append(f"{api_id}: field {field_id} examples.{example} does not satisfy its declared {constraint} boundary")
|
|
331
|
+
if base_value is not MISSING:
|
|
332
|
+
type_matches = {
|
|
333
|
+
"string": isinstance(base_value, str),
|
|
334
|
+
"integer": isinstance(base_value, int) and not isinstance(base_value, bool),
|
|
335
|
+
"number": isinstance(base_value, (int, float)) and not isinstance(base_value, bool),
|
|
336
|
+
"boolean": isinstance(base_value, bool),
|
|
337
|
+
"array": isinstance(base_value, list),
|
|
338
|
+
"object": isinstance(base_value, dict),
|
|
339
|
+
}
|
|
340
|
+
if base_value is None:
|
|
341
|
+
if field.get("nullable") is False:
|
|
342
|
+
errors.append(f"{api_id}: field {field_id} valid_request cannot be null when nullable is false")
|
|
343
|
+
elif value_type in TYPES and not type_matches[value_type]:
|
|
344
|
+
errors.append(f"{api_id}: field {field_id} valid_request value does not match declared type {value_type}")
|
|
345
|
+
else:
|
|
346
|
+
if isinstance(constraints.get("enum"), list) and base_value not in constraints["enum"]:
|
|
347
|
+
errors.append(f"{api_id}: field {field_id} valid_request violates enum")
|
|
348
|
+
if value_type == "string":
|
|
349
|
+
if isinstance(constraints.get("min_length"), int) and len(base_value) < constraints["min_length"]:
|
|
350
|
+
errors.append(f"{api_id}: field {field_id} valid_request is below min_length")
|
|
351
|
+
if isinstance(constraints.get("max_length"), int) and len(base_value) > constraints["max_length"]:
|
|
352
|
+
errors.append(f"{api_id}: field {field_id} valid_request is above max_length")
|
|
353
|
+
if value_type in {"integer", "number"}:
|
|
354
|
+
if isinstance(constraints.get("minimum"), (int, float)) and base_value < constraints["minimum"]:
|
|
355
|
+
errors.append(f"{api_id}: field {field_id} valid_request is below minimum")
|
|
356
|
+
if isinstance(constraints.get("maximum"), (int, float)) and base_value > constraints["maximum"]:
|
|
357
|
+
errors.append(f"{api_id}: field {field_id} valid_request is above maximum")
|
|
358
|
+
if value_type == "array":
|
|
359
|
+
if isinstance(constraints.get("min_items"), int) and len(base_value) < constraints["min_items"]:
|
|
360
|
+
errors.append(f"{api_id}: field {field_id} valid_request has too few array items")
|
|
361
|
+
if isinstance(constraints.get("max_items"), int) and len(base_value) > constraints["max_items"]:
|
|
362
|
+
errors.append(f"{api_id}: field {field_id} valid_request has too many array items")
|
|
363
|
+
|
|
364
|
+
|
|
365
|
+
def validate_schema(api: dict[str, Any], errors: list[str]) -> None:
|
|
366
|
+
api_id = str(api.get("id", "API-unknown"))
|
|
367
|
+
data = api.get("data") if isinstance(api.get("data"), dict) else {}
|
|
368
|
+
schema = data.get("test_schema")
|
|
369
|
+
if not isinstance(schema, dict):
|
|
370
|
+
errors.append(f"{api_id}: API contract requires test_schema for complete field/error tests")
|
|
371
|
+
return
|
|
372
|
+
if schema.get("schema_version") != 1:
|
|
373
|
+
errors.append(f"{api_id}: test_schema.schema_version must be 1")
|
|
374
|
+
if not isinstance(schema.get("method"), str) or not re.fullmatch(r"[A-Z]+", schema["method"]):
|
|
375
|
+
errors.append(f"{api_id}: test_schema.method must be uppercase HTTP method")
|
|
376
|
+
if not isinstance(schema.get("path"), str) or not schema["path"].startswith("/"):
|
|
377
|
+
errors.append(f"{api_id}: test_schema.path must start with /")
|
|
378
|
+
request = schema.get("valid_request")
|
|
379
|
+
if not isinstance(request, dict):
|
|
380
|
+
errors.append(f"{api_id}: test_schema.valid_request must be an object")
|
|
381
|
+
else:
|
|
382
|
+
for location in LOCATIONS:
|
|
383
|
+
if location in request and not isinstance(request[location], dict):
|
|
384
|
+
errors.append(f"{api_id}: valid_request.{location} must be an object")
|
|
385
|
+
success = schema.get("success")
|
|
386
|
+
if not isinstance(success, dict) or not isinstance(success.get("http_status"), int) or not 100 <= success["http_status"] <= 599:
|
|
387
|
+
errors.append(f"{api_id}: test_schema.success.http_status must be an HTTP status")
|
|
388
|
+
preconditions = schema.get("preconditions")
|
|
389
|
+
if not isinstance(preconditions, list) or not preconditions or not all(isinstance(item, str) and item.strip() for item in preconditions):
|
|
390
|
+
errors.append(f"{api_id}: test_schema.preconditions must be a non-empty string array")
|
|
391
|
+
fields = schema.get("fields")
|
|
392
|
+
if not isinstance(fields, list):
|
|
393
|
+
errors.append(f"{api_id}: test_schema.fields must be an array")
|
|
394
|
+
fields = []
|
|
395
|
+
seen: set[str] = set()
|
|
396
|
+
for field in fields:
|
|
397
|
+
validate_field(api_id, schema, field, seen, errors)
|
|
398
|
+
catalog = schema.get("errors")
|
|
399
|
+
if not isinstance(catalog, list):
|
|
400
|
+
errors.append(f"{api_id}: test_schema.errors must be an array (use [] only when no error contract exists)")
|
|
401
|
+
return
|
|
402
|
+
error_ids: set[str] = set()
|
|
403
|
+
for item in catalog:
|
|
404
|
+
if not isinstance(item, dict):
|
|
405
|
+
errors.append(f"{api_id}: error catalogue entries must be objects")
|
|
406
|
+
continue
|
|
407
|
+
error_id = item.get("error_key")
|
|
408
|
+
if not isinstance(error_id, str) or not error_id.strip():
|
|
409
|
+
errors.append(f"{api_id}: error catalogue error_key is required")
|
|
410
|
+
elif error_id in error_ids:
|
|
411
|
+
errors.append(f"{api_id}: duplicate error catalogue id {error_id}")
|
|
412
|
+
else:
|
|
413
|
+
error_ids.add(error_id)
|
|
414
|
+
if not isinstance(item.get("code"), str) or not item["code"].strip():
|
|
415
|
+
errors.append(f"{api_id}: error catalogue {error_id or '<unknown>'} code is required")
|
|
416
|
+
if not isinstance(item.get("http_status"), int) or not 100 <= item["http_status"] <= 599:
|
|
417
|
+
errors.append(f"{api_id}: error catalogue {error_id or '<unknown>'} http_status is invalid")
|
|
418
|
+
covers = item.get("covers")
|
|
419
|
+
scenario = item.get("scenario")
|
|
420
|
+
if bool(covers) == bool(scenario):
|
|
421
|
+
errors.append(f"{api_id}: error catalogue {error_id or '<unknown>'} needs exactly one of covers or scenario")
|
|
422
|
+
if covers is not None and (not isinstance(covers, list) or not covers or not all(isinstance(value, str) and value for value in covers)):
|
|
423
|
+
errors.append(f"{api_id}: error catalogue {error_id or '<unknown>'} covers must be a non-empty string array")
|
|
424
|
+
if scenario is not None:
|
|
425
|
+
if not isinstance(scenario, dict):
|
|
426
|
+
errors.append(f"{api_id}: error catalogue {error_id or '<unknown>'} scenario must be an object")
|
|
427
|
+
else:
|
|
428
|
+
pre = scenario.get("preconditions")
|
|
429
|
+
mutations = scenario.get("request_mutations")
|
|
430
|
+
if not isinstance(pre, list) or not pre or not all(isinstance(value, str) and value.strip() for value in pre):
|
|
431
|
+
errors.append(f"{api_id}: error catalogue {error_id or '<unknown>'} scenario.preconditions must be non-empty")
|
|
432
|
+
if not isinstance(mutations, list) or not mutations:
|
|
433
|
+
errors.append(f"{api_id}: error catalogue {error_id or '<unknown>'} scenario.request_mutations must be non-empty")
|
|
434
|
+
|
|
435
|
+
|
|
436
|
+
def valid_values(field: dict[str, Any], base_value: Any) -> list[Any]:
|
|
437
|
+
constraints = field.get("constraints") if isinstance(field.get("constraints"), dict) else {}
|
|
438
|
+
examples = field.get("examples") if isinstance(field.get("examples"), dict) else {}
|
|
439
|
+
values: list[Any] = [base_value]
|
|
440
|
+
value_type = field.get("type")
|
|
441
|
+
if value_type == "string":
|
|
442
|
+
if "min_length" in constraints:
|
|
443
|
+
values.append(examples.get("at_min_length", "x" * constraints["min_length"]))
|
|
444
|
+
if "max_length" in constraints:
|
|
445
|
+
values.append(examples.get("at_max_length", "x" * constraints["max_length"]))
|
|
446
|
+
if value_type in {"integer", "number"} and "enum" not in constraints:
|
|
447
|
+
if "minimum" in constraints:
|
|
448
|
+
values.append(examples.get("at_minimum", constraints["minimum"]))
|
|
449
|
+
if "maximum" in constraints:
|
|
450
|
+
values.append(examples.get("at_maximum", constraints["maximum"]))
|
|
451
|
+
if value_type == "array" and isinstance(base_value, list):
|
|
452
|
+
if "min_items" in constraints:
|
|
453
|
+
values.append(examples.get("at_min_items", base_value[: constraints["min_items"]]))
|
|
454
|
+
if "max_items" in constraints:
|
|
455
|
+
fill = list(base_value) or [examples.get("array_item", "__devflow_item__")]
|
|
456
|
+
values.append(examples.get("at_max_items", (fill * (constraints["max_items"] + 1))[: constraints["max_items"]]))
|
|
457
|
+
if isinstance(constraints.get("enum"), list):
|
|
458
|
+
values.extend(constraints["enum"])
|
|
459
|
+
if value_type == "boolean":
|
|
460
|
+
values.extend([False, True])
|
|
461
|
+
if isinstance(field.get("valid_values"), list):
|
|
462
|
+
values.extend(field["valid_values"])
|
|
463
|
+
if field.get("required") is False:
|
|
464
|
+
values.append(MISSING)
|
|
465
|
+
if field.get("nullable") is True:
|
|
466
|
+
values.append(None)
|
|
467
|
+
return unique(values)
|
|
468
|
+
|
|
469
|
+
|
|
470
|
+
def pairwise(domains: list[list[Any]]) -> list[list[Any]]:
|
|
471
|
+
"""Small deterministic IPOG-style generator for valid equivalence classes."""
|
|
472
|
+
if not domains:
|
|
473
|
+
return [[]]
|
|
474
|
+
if len(domains) == 1:
|
|
475
|
+
return [[value] for value in domains[0]]
|
|
476
|
+
rows = [list(pair) for pair in product(domains[0], domains[1])]
|
|
477
|
+
for index in range(2, len(domains)):
|
|
478
|
+
domain = domains[index]
|
|
479
|
+
uncovered = {(prior, prior_value, value) for prior in range(index) for prior_value in domains[prior] for value in domain}
|
|
480
|
+
for row in rows:
|
|
481
|
+
best = max(domain, key=lambda value: sum((prior, row[prior], value) in uncovered for prior in range(index)))
|
|
482
|
+
row.append(best)
|
|
483
|
+
for prior in range(index):
|
|
484
|
+
uncovered.discard((prior, row[prior], best))
|
|
485
|
+
while uncovered:
|
|
486
|
+
prior, prior_value, value = sorted(uncovered, key=lambda item: (item[0], stable_value(item[1]), stable_value(item[2])))[0]
|
|
487
|
+
row = [domain_values[0] for domain_values in domains[:index]] + [value]
|
|
488
|
+
row[prior] = prior_value
|
|
489
|
+
rows.append(row)
|
|
490
|
+
for previous in range(index):
|
|
491
|
+
uncovered.discard((previous, row[previous], value))
|
|
492
|
+
return rows
|
|
493
|
+
|
|
494
|
+
|
|
495
|
+
def request_from_mutations(schema: dict[str, Any], mutations: list[dict[str, Any]]) -> dict[str, Any]:
|
|
496
|
+
request = copy.deepcopy(schema["valid_request"])
|
|
497
|
+
for mutation in mutations:
|
|
498
|
+
if not isinstance(mutation, dict):
|
|
499
|
+
raise ValueError("request mutation must be an object")
|
|
500
|
+
location = mutation.get("location")
|
|
501
|
+
pointer = mutation.get("pointer")
|
|
502
|
+
operation = mutation.get("op")
|
|
503
|
+
if location not in LOCATIONS or not isinstance(pointer, str) or operation not in {"replace", "remove"}:
|
|
504
|
+
raise ValueError("request mutation requires location, pointer and replace/remove op")
|
|
505
|
+
container = request.setdefault(location, {})
|
|
506
|
+
if operation == "remove":
|
|
507
|
+
pointer_remove(container, pointer)
|
|
508
|
+
else:
|
|
509
|
+
pointer_set(container, pointer, copy.deepcopy(mutation.get("value")))
|
|
510
|
+
return request
|
|
511
|
+
|
|
512
|
+
|
|
513
|
+
def request_for_field(schema: dict[str, Any], field: dict[str, Any], value: Any) -> dict[str, Any]:
|
|
514
|
+
request = copy.deepcopy(schema["valid_request"])
|
|
515
|
+
container = request.setdefault(field["location"], {})
|
|
516
|
+
if value is MISSING:
|
|
517
|
+
pointer_remove(container, field["pointer"])
|
|
518
|
+
else:
|
|
519
|
+
pointer_set(container, field["pointer"], copy.deepcopy(value))
|
|
520
|
+
return request
|
|
521
|
+
|
|
522
|
+
|
|
523
|
+
def request_execution(schema: dict[str, Any], request: dict[str, Any], expect: dict[str, Any]) -> dict[str, Any]:
|
|
524
|
+
return {
|
|
525
|
+
"protocol": "http",
|
|
526
|
+
"request": {
|
|
527
|
+
"method": schema["method"],
|
|
528
|
+
"path": schema["path"],
|
|
529
|
+
"path_params": request.get("path", {}),
|
|
530
|
+
"query": request.get("query", {}),
|
|
531
|
+
"headers": request.get("headers", {}),
|
|
532
|
+
"body": request.get("body", {}),
|
|
533
|
+
},
|
|
534
|
+
"expect": expect,
|
|
535
|
+
}
|
|
536
|
+
|
|
537
|
+
|
|
538
|
+
def tc_candidate(api: dict[str, Any], graph: dict[str, Any], key: str, obligation_refs: list[str], execution: dict[str, Any], title: str, preconditions: list[str], oracle: str) -> dict[str, Any]:
|
|
539
|
+
data = api.get("data", {})
|
|
540
|
+
requirement_refs = list(data.get("requirement_refs", []))
|
|
541
|
+
acceptance_refs = [
|
|
542
|
+
node["id"]
|
|
543
|
+
for node in graph.get("nodes", [])
|
|
544
|
+
if isinstance(node, dict) and node.get("kind") == "acceptance" and isinstance(node.get("data"), dict) and node["data"].get("requirement_ref") in requirement_refs
|
|
545
|
+
]
|
|
546
|
+
return {
|
|
547
|
+
"generated_case_key": key,
|
|
548
|
+
"test_obligation_refs": obligation_refs,
|
|
549
|
+
"acceptance_refs": acceptance_refs,
|
|
550
|
+
"requirement_refs": requirement_refs,
|
|
551
|
+
"test_type": "api",
|
|
552
|
+
"target": f"{execution['request']['method']} {execution['request']['path']}: {title}",
|
|
553
|
+
"preconditions": preconditions,
|
|
554
|
+
"steps": ["发送 execution.request 中定义的 HTTP 请求"],
|
|
555
|
+
"oracle": oracle,
|
|
556
|
+
"automation_mode": "automated",
|
|
557
|
+
"api_refs": [api["id"]],
|
|
558
|
+
"execution": execution,
|
|
559
|
+
}
|
|
560
|
+
|
|
561
|
+
|
|
562
|
+
def expected_error(entry: dict[str, Any]) -> dict[str, Any]:
|
|
563
|
+
return {"http_status": entry["http_status"], "error_code": entry["code"], "error_code_pointer": "/error/code"}
|
|
564
|
+
|
|
565
|
+
|
|
566
|
+
def derive_api_cases(api: dict[str, Any], graph: dict[str, Any], errors: list[str]) -> list[dict[str, Any]]:
|
|
567
|
+
api_id = api["id"]
|
|
568
|
+
schema = api["data"]["test_schema"]
|
|
569
|
+
fields = schema.get("fields", [])
|
|
570
|
+
catalog = schema.get("errors", [])
|
|
571
|
+
mapping: dict[str, dict[str, Any]] = {}
|
|
572
|
+
used_mapping: set[str] = set()
|
|
573
|
+
scenario_errors: list[dict[str, Any]] = []
|
|
574
|
+
for error in catalog:
|
|
575
|
+
if not isinstance(error, dict):
|
|
576
|
+
continue
|
|
577
|
+
if isinstance(error.get("covers"), list):
|
|
578
|
+
for cover in error["covers"]:
|
|
579
|
+
if cover in mapping:
|
|
580
|
+
errors.append(f"{api_id}: partition {cover} is mapped by more than one error code")
|
|
581
|
+
mapping[cover] = error
|
|
582
|
+
elif isinstance(error.get("scenario"), dict):
|
|
583
|
+
scenario_errors.append(error)
|
|
584
|
+
result: list[dict[str, Any]] = []
|
|
585
|
+
number = 0
|
|
586
|
+
|
|
587
|
+
# Valid equivalence classes first. This is the only place pairwise mixes
|
|
588
|
+
# fields, and all values in its domains are valid by construction.
|
|
589
|
+
domains: list[list[Any]] = []
|
|
590
|
+
valid_field_data: list[tuple[dict[str, Any], Any]] = []
|
|
591
|
+
for field in fields:
|
|
592
|
+
base = pointer_get(schema["valid_request"].get(field["location"], {}), field["pointer"])
|
|
593
|
+
domains.append(valid_values(field, base))
|
|
594
|
+
valid_field_data.append((field, base))
|
|
595
|
+
for row in pairwise(domains):
|
|
596
|
+
request = copy.deepcopy(schema["valid_request"])
|
|
597
|
+
labels: list[str] = []
|
|
598
|
+
for (field, _base), value in zip(valid_field_data, row):
|
|
599
|
+
container = request.setdefault(field["location"], {})
|
|
600
|
+
if value is MISSING:
|
|
601
|
+
pointer_remove(container, field["pointer"])
|
|
602
|
+
labels.append(f"{field['field_key']}=omit")
|
|
603
|
+
else:
|
|
604
|
+
pointer_set(container, field["pointer"], copy.deepcopy(value))
|
|
605
|
+
labels.append(f"{field['field_key']}=valid")
|
|
606
|
+
number += 1
|
|
607
|
+
key = case_key(api_id, "pairwise", number)
|
|
608
|
+
refs = [obligation_id(api_id, "pairwise", f"{number:04d}")]
|
|
609
|
+
execution = request_execution(schema, request, {"http_status": schema["success"]["http_status"]})
|
|
610
|
+
result.append({
|
|
611
|
+
"key": key,
|
|
612
|
+
"category": "pairwise",
|
|
613
|
+
"title": "valid pairwise: " + ", ".join(labels),
|
|
614
|
+
"obligation_refs": refs,
|
|
615
|
+
"test_case": tc_candidate(api, graph, key, refs, execution, "valid pairwise", list(schema["preconditions"]), f"HTTP {schema['success']['http_status']}"),
|
|
616
|
+
})
|
|
617
|
+
|
|
618
|
+
def add_negative(field: dict[str, Any], partition: str, value: Any, label: str) -> None:
|
|
619
|
+
nonlocal number
|
|
620
|
+
map_key = f"{field['field_key']}:{partition}"
|
|
621
|
+
error = mapping.get(map_key)
|
|
622
|
+
if error is None:
|
|
623
|
+
errors.append(error_code_error(api_id, field["field_key"], partition))
|
|
624
|
+
return
|
|
625
|
+
used_mapping.add(map_key)
|
|
626
|
+
number += 1
|
|
627
|
+
key = case_key(api_id, "negative", number)
|
|
628
|
+
ref = obligation_id(api_id, field["field_key"], partition)
|
|
629
|
+
execution = request_execution(schema, request_for_field(schema, field, value), expected_error(error))
|
|
630
|
+
result.append({
|
|
631
|
+
"key": key,
|
|
632
|
+
"category": "negative",
|
|
633
|
+
"title": f"{field['field_key']}: {label}",
|
|
634
|
+
"obligation_refs": [ref],
|
|
635
|
+
"test_case": tc_candidate(api, graph, key, [ref], execution, f"{field['field_key']} {label}", list(schema["preconditions"]), f"HTTP {error['http_status']} and error code {error['code']}"),
|
|
636
|
+
})
|
|
637
|
+
|
|
638
|
+
for field in fields:
|
|
639
|
+
constraints = field.get("constraints", {})
|
|
640
|
+
examples = field.get("examples", {}) if isinstance(field.get("examples"), dict) else {}
|
|
641
|
+
value_type = field["type"]
|
|
642
|
+
if field.get("required"):
|
|
643
|
+
add_negative(field, "missing", MISSING, "required value omitted")
|
|
644
|
+
if not field.get("nullable"):
|
|
645
|
+
add_negative(field, "null", None, "null value")
|
|
646
|
+
add_negative(field, "wrong_type", wrong_type(value_type), "wrong type")
|
|
647
|
+
if value_type == "string":
|
|
648
|
+
if "min_length" in constraints and constraints["min_length"] > 0:
|
|
649
|
+
add_negative(field, "below_min_length", examples.get("below_min_length", "x" * (constraints["min_length"] - 1)), "below minimum length")
|
|
650
|
+
if "max_length" in constraints:
|
|
651
|
+
add_negative(field, "above_max_length", examples.get("above_max_length", "x" * (constraints["max_length"] + 1)), "above maximum length")
|
|
652
|
+
if "format" in constraints:
|
|
653
|
+
invalid = examples.get("invalid_format", standard_invalid_format(str(constraints["format"])))
|
|
654
|
+
if invalid is None:
|
|
655
|
+
errors.append(f"{api_id}: field {field['field_key']} format {constraints['format']} requires examples.invalid_format")
|
|
656
|
+
else:
|
|
657
|
+
add_negative(field, "invalid_format", invalid, "invalid format")
|
|
658
|
+
if "pattern" in constraints:
|
|
659
|
+
add_negative(field, "invalid_pattern", examples.get("invalid_pattern"), "invalid pattern")
|
|
660
|
+
if value_type in {"integer", "number"}:
|
|
661
|
+
if "minimum" in constraints:
|
|
662
|
+
add_negative(field, "below_minimum", examples.get("below_minimum", constraints["minimum"] - 1), "below minimum")
|
|
663
|
+
if "maximum" in constraints:
|
|
664
|
+
add_negative(field, "above_maximum", examples.get("above_maximum", constraints["maximum"] + 1), "above maximum")
|
|
665
|
+
if value_type == "array":
|
|
666
|
+
base = pointer_get(schema["valid_request"].get(field["location"], {}), field["pointer"])
|
|
667
|
+
if "min_items" in constraints and constraints["min_items"] > 0:
|
|
668
|
+
add_negative(field, "below_min_items", list(base)[: constraints["min_items"] - 1], "below minimum item count")
|
|
669
|
+
if "max_items" in constraints:
|
|
670
|
+
fill = list(base) or [examples.get("array_item", "__devflow_item__")]
|
|
671
|
+
expanded = (fill * (constraints["max_items"] + 1))[: constraints["max_items"] + 1]
|
|
672
|
+
add_negative(field, "above_max_items", expanded, "above maximum item count")
|
|
673
|
+
if "enum" in constraints:
|
|
674
|
+
add_negative(field, "invalid_enum", examples.get("invalid_enum"), "value outside enum")
|
|
675
|
+
|
|
676
|
+
for mapped_partition in sorted(set(mapping) - used_mapping):
|
|
677
|
+
errors.append(f"{api_id}: error mapping {mapped_partition} does not name a derivable negative partition")
|
|
678
|
+
|
|
679
|
+
for error in scenario_errors:
|
|
680
|
+
scenario = error["scenario"]
|
|
681
|
+
try:
|
|
682
|
+
request = request_from_mutations(schema, scenario["request_mutations"])
|
|
683
|
+
except (KeyError, TypeError, ValueError) as failure:
|
|
684
|
+
errors.append(f"{api_id}: scenario error {error.get('id', '<unknown>')} has invalid request mutation: {failure}")
|
|
685
|
+
continue
|
|
686
|
+
number += 1
|
|
687
|
+
key = case_key(api_id, "error", number)
|
|
688
|
+
ref = obligation_id(api_id, "error", str(error["error_key"]))
|
|
689
|
+
execution = request_execution(schema, request, expected_error(error))
|
|
690
|
+
preconditions = [*schema["preconditions"], *scenario["preconditions"]]
|
|
691
|
+
result.append({
|
|
692
|
+
"key": key,
|
|
693
|
+
"category": "error",
|
|
694
|
+
"title": f"error code {error['code']}",
|
|
695
|
+
"obligation_refs": [ref],
|
|
696
|
+
"test_case": tc_candidate(api, graph, key, [ref], execution, f"error {error['code']}", preconditions, f"HTTP {error['http_status']} and error code {error['code']}"),
|
|
697
|
+
})
|
|
698
|
+
return result
|
|
699
|
+
|
|
700
|
+
|
|
701
|
+
def compile_matrix(graph: dict[str, Any], feature: str) -> tuple[dict[str, Any], list[str]]:
|
|
702
|
+
errors: list[str] = []
|
|
703
|
+
APIs = api_nodes(graph)
|
|
704
|
+
for api in APIs:
|
|
705
|
+
validate_schema(api, errors)
|
|
706
|
+
cases: list[dict[str, Any]] = []
|
|
707
|
+
if not errors:
|
|
708
|
+
for api in APIs:
|
|
709
|
+
cases.extend(derive_api_cases(api, graph, errors))
|
|
710
|
+
source = {"api_contract_fingerprint": api_fingerprint(graph), "api_ids": [node.get("id") for node in APIs]}
|
|
711
|
+
case_manifest = [
|
|
712
|
+
{"key": item["key"], "obligation_refs": item["obligation_refs"], "test_case": item["test_case"]}
|
|
713
|
+
for item in cases
|
|
714
|
+
]
|
|
715
|
+
matrix = {
|
|
716
|
+
"schema_version": 1,
|
|
717
|
+
"origin": "core_api_test_matrix",
|
|
718
|
+
"feature": feature,
|
|
719
|
+
"source": source,
|
|
720
|
+
"case_set_fingerprint": digest(case_manifest),
|
|
721
|
+
"cases": cases,
|
|
722
|
+
"errors": errors,
|
|
723
|
+
}
|
|
724
|
+
return matrix, errors
|
|
725
|
+
|
|
726
|
+
|
|
727
|
+
def render_markdown(matrix: dict[str, Any]) -> str:
|
|
728
|
+
lines = ["# 完整 API 验收测试用例", "", f"- Feature: `{matrix['feature']}`", f"- API 契约指纹: `{matrix['source']['api_contract_fingerprint']}`", f"- 用例集指纹: `{matrix['case_set_fingerprint']}`", f"- 用例数: {len(matrix['cases'])}", "", "| 用例键 | 类型 | 覆盖义务 | 请求/预期 |", "|---|---|---|---|"]
|
|
729
|
+
for item in matrix["cases"]:
|
|
730
|
+
execution = item["test_case"]["execution"]
|
|
731
|
+
request = execution["request"]
|
|
732
|
+
expect = execution["expect"]
|
|
733
|
+
expected = f"HTTP {expect['http_status']}" + (f", {expect['error_code']}" if "error_code" in expect else "")
|
|
734
|
+
lines.append(f"| `{item['key']}` | {item['category']} | {', '.join(f'`{ref}`' for ref in item['obligation_refs'])} | `{request['method']} {request['path']}` → {expected} |")
|
|
735
|
+
if matrix["errors"]:
|
|
736
|
+
lines.extend(["", "## 契约缺口(不可冻结)", "", *[f"- {error}" for error in matrix["errors"]]])
|
|
737
|
+
lines.extend(["", "## 执行方式", "", "每条用例均含 `execution.protocol=http`、完整请求和可观察预期。导入后的 TC ID 写入同目录的 `api-acceptance-bindings.template.json`;为该文件填写测试环境基地址和鉴权环境变量后,`run-acceptance-tests` 可逐条执行。需要专用夹具的数据前置条件不会被省略,必须由绑定中的 setup 命令或测试环境准备满足。", ""])
|
|
738
|
+
return "\n".join(lines)
|
|
739
|
+
|
|
740
|
+
|
|
741
|
+
def binding_template(matrix: dict[str, Any]) -> dict[str, Any]:
|
|
742
|
+
cases = []
|
|
743
|
+
for item in matrix["cases"]:
|
|
744
|
+
tc_id = item.get("tc_id")
|
|
745
|
+
if isinstance(tc_id, str) and tc_id:
|
|
746
|
+
cases.append({"tc_id": tc_id, "mode": "automated", "adapter": {"kind": "http", "base_url_env": "DEVFLOW_TEST_BASE_URL", "headers_env": {"Authorization": "DEVFLOW_TEST_AUTHORIZATION"}}})
|
|
747
|
+
else:
|
|
748
|
+
cases.append({"generated_case_key": item["key"], "mode": "automated", "adapter": {"kind": "http", "base_url_env": "DEVFLOW_TEST_BASE_URL", "headers_env": {"Authorization": "DEVFLOW_TEST_AUTHORIZATION"}}})
|
|
749
|
+
return {"schema_version": 1, "origin": "core_api_test_matrix_template", "feature": matrix["feature"], "case_set_fingerprint": matrix["case_set_fingerprint"], "cases": cases}
|
|
750
|
+
|
|
751
|
+
|
|
752
|
+
def validate_imported(matrix: dict[str, Any], graph: dict[str, Any], errors: list[str]) -> None:
|
|
753
|
+
expected, compile_errors = compile_matrix(graph, matrix.get("feature", ""))
|
|
754
|
+
errors.extend(compile_errors)
|
|
755
|
+
if matrix.get("origin") != "core_api_test_matrix" or matrix.get("schema_version") != 1:
|
|
756
|
+
errors.append("API test matrix identity is invalid")
|
|
757
|
+
return
|
|
758
|
+
if matrix.get("source") != expected.get("source"):
|
|
759
|
+
errors.append("API test matrix is stale for the current API contract")
|
|
760
|
+
if matrix.get("case_set_fingerprint") != expected.get("case_set_fingerprint"):
|
|
761
|
+
errors.append("API test matrix cases differ from the deterministic compiler output")
|
|
762
|
+
expected_by_key = {item["key"]: item for item in expected.get("cases", [])}
|
|
763
|
+
actual_cases = matrix.get("cases") if isinstance(matrix.get("cases"), list) else []
|
|
764
|
+
actual_by_key = {item.get("key"): item for item in actual_cases if isinstance(item, dict) and isinstance(item.get("key"), str)}
|
|
765
|
+
if set(actual_by_key) != set(expected_by_key):
|
|
766
|
+
errors.append("API test matrix does not contain exactly the derived test cases")
|
|
767
|
+
return
|
|
768
|
+
graph_cases = {
|
|
769
|
+
node.get("data", {}).get("generated_case_key"): node
|
|
770
|
+
for node in graph.get("nodes", [])
|
|
771
|
+
if isinstance(node, dict) and node.get("kind") == "test_case" and isinstance(node.get("data"), dict) and isinstance(node["data"].get("generated_case_key"), str)
|
|
772
|
+
}
|
|
773
|
+
for key, expected_case in expected_by_key.items():
|
|
774
|
+
actual = actual_by_key[key]
|
|
775
|
+
tc_id = actual.get("tc_id")
|
|
776
|
+
node = graph_cases.get(key)
|
|
777
|
+
if not isinstance(tc_id, str) or not tc_id:
|
|
778
|
+
errors.append(f"{key}: has not been imported as a TC candidate")
|
|
779
|
+
continue
|
|
780
|
+
if not isinstance(node, dict) or node.get("id") != tc_id:
|
|
781
|
+
errors.append(f"{key}: imported TC does not match matrix tc_id")
|
|
782
|
+
continue
|
|
783
|
+
data = node.get("data", {})
|
|
784
|
+
if data.get("test_obligation_refs") != expected_case["obligation_refs"]:
|
|
785
|
+
errors.append(f"{key}: TC obligation refs differ from the deterministic matrix")
|
|
786
|
+
if data.get("execution") != expected_case["test_case"]["execution"]:
|
|
787
|
+
errors.append(f"{key}: TC execution definition differs from the deterministic matrix")
|
|
788
|
+
|
|
789
|
+
|
|
790
|
+
def run(action: str, root: Path, feature: str, write: bool) -> dict[str, Any]:
|
|
791
|
+
governance_path, matrix_path, markdown_path, template_path = paths(root, feature)
|
|
792
|
+
errors: list[str] = []
|
|
793
|
+
graph = read_json(governance_path, errors, "requirement graph")
|
|
794
|
+
if graph is None:
|
|
795
|
+
return {"ok": False, "feature": feature, "errors": errors}
|
|
796
|
+
matrix, compile_errors = compile_matrix(graph, feature)
|
|
797
|
+
if action == "check":
|
|
798
|
+
return {"ok": not compile_errors, "feature": feature, "errors": compile_errors, "case_count": len(matrix["cases"]), "case_set_fingerprint": matrix["case_set_fingerprint"]}
|
|
799
|
+
if action == "compile":
|
|
800
|
+
if write:
|
|
801
|
+
write_json(matrix_path, matrix)
|
|
802
|
+
write_text(markdown_path, render_markdown(matrix))
|
|
803
|
+
write_json(template_path, binding_template(matrix))
|
|
804
|
+
return {"ok": not compile_errors, "feature": feature, "errors": compile_errors, "paths": {"matrix": str(matrix_path.relative_to(root)), "view": str(markdown_path.relative_to(root)), "binding_template": str(template_path.relative_to(root))}, "case_count": len(matrix["cases"]), "case_set_fingerprint": matrix["case_set_fingerprint"]}
|
|
805
|
+
persisted = read_json(matrix_path, errors, "API test matrix")
|
|
806
|
+
if persisted is not None:
|
|
807
|
+
validate_imported(persisted, graph, errors)
|
|
808
|
+
return {"ok": not errors, "feature": feature, "errors": errors, "case_count": len(matrix["cases"]), "case_set_fingerprint": matrix["case_set_fingerprint"]}
|
|
809
|
+
|
|
810
|
+
|
|
811
|
+
def main() -> int:
|
|
812
|
+
parser = argparse.ArgumentParser()
|
|
813
|
+
parser.add_argument("action", choices=("check", "compile", "validate"))
|
|
814
|
+
parser.add_argument("--feature", required=True)
|
|
815
|
+
parser.add_argument("--write", action="store_true")
|
|
816
|
+
parser.add_argument("--json", action="store_true")
|
|
817
|
+
args = parser.parse_args()
|
|
818
|
+
if not FEATURE_RE.fullmatch(args.feature):
|
|
819
|
+
result = {"ok": False, "errors": ["invalid feature"]}
|
|
820
|
+
elif args.action == "compile" and not args.write:
|
|
821
|
+
result = {"ok": False, "errors": ["compile requires --write"]}
|
|
822
|
+
else:
|
|
823
|
+
result = run(args.action, Path.cwd().resolve(), args.feature, args.write)
|
|
824
|
+
print(json.dumps(result, ensure_ascii=False, indent=2))
|
|
825
|
+
return 0
|
|
826
|
+
|
|
827
|
+
|
|
828
|
+
if __name__ == "__main__":
|
|
829
|
+
raise SystemExit(main())
|