@guanlin4924/devflow-pi 0.1.19 → 0.1.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -0
- package/package.json +3 -3
- package/plugins/devflow-core/extensions/devflow-core.ts +142 -2
- package/plugins/devflow-core/skills/workflow-orchestrator/scripts/acceptance_test_runner.py +394 -0
- package/plugins/devflow-core/skills/workflow-orchestrator/scripts/acceptance_test_suite.rb +86 -0
- package/plugins/devflow-core/skills/workflow-orchestrator/scripts/advance_state.rb +192 -1
- package/plugins/devflow-core/skills/workflow-orchestrator/scripts/api_test_matrix.py +829 -0
- package/plugins/devflow-core/skills/workflow-orchestrator/scripts/execution_contract.rb +7 -5
- package/plugins/devflow-core/skills/workflow-orchestrator/scripts/init_state.rb +11 -1
- package/plugins/devflow-core/skills/workflow-orchestrator/scripts/inspect_state.rb +5 -1
- package/plugins/devflow-core/skills/workflow-orchestrator/scripts/machine_evidence.rb +39 -3
- package/plugins/devflow-core/skills/workflow-orchestrator/scripts/requirement_graph.rb +309 -23
- package/plugins/devflow-core/skills/workflow-orchestrator/scripts/tdd_evidence.py +311 -0
- package/plugins/devflow-core/skills/workflow-orchestrator/scripts/test_command_policy.py +53 -0
- package/plugins/devflow-core/skills/workflow-orchestrator/scripts/test_task_binding_check.py +254 -0
- package/plugins/devflow-core/skills/workflow-orchestrator/templates/feature-development.yaml +109 -4
- package/plugins/devflow-core/skills/workflow-orchestrator/templates/uat-remediation.yaml +21 -1
- package/plugins/devflow-design/skills/workflow-api-contract/SKILL.md +38 -0
- package/plugins/devflow-implementation/agents/implementation-worker.md +3 -0
- package/plugins/devflow-implementation/skills/workflow-change-plan/SKILL.md +7 -0
- package/plugins/devflow-implementation/skills/workflow-change-plan/references/output-template.md +2 -0
- package/plugins/devflow-implementation/skills/workflow-execute-phase/SKILL.md +3 -3
- package/plugins/devflow-quality/skills/workflow-acceptance-test-design/SKILL.md +63 -0
- package/plugins/devflow-quality/skills/workflow-acceptance-test-design/agents/openai.yaml +4 -0
- package/plugins/devflow-quality/skills/workflow-generate-mainflow-cases/SKILL.md +7 -6
- package/plugins/devflow-quality/skills/workflow-generate-mainflow-cases/references/intermediate-format.md +3 -0
- package/plugins/devflow-quality/skills/workflow-generate-mainflow-cases/scripts/compile_dsl.py +49 -3
- package/plugins/devflow-quality/skills/workflow-run-acceptance-tests/SKILL.md +48 -0
- package/plugins/devflow-quality/skills/workflow-run-acceptance-tests/agents/openai.yaml +4 -0
- package/plugins/devflow-quality/skills/workflow-verify-work/SKILL.md +9 -3
package/README.md
CHANGED
|
@@ -119,6 +119,25 @@ bash .skills/install.sh --apply --force-config
|
|
|
119
119
|
|
|
120
120
|
`/devflow start` 会先进入安全的需求梳理:可澄清并只读检查,不写入项目。用户随后直接补充需求,原文会被带入入口分诊并启动当前步骤。每个步骤结束后,系统展示产物和证据;用户明确回复“批准执行”才会推进并运行下一步,回复修改意见或“拒绝执行”则保持原步骤。普通需求描述绝不会自动确认 checkpoint。`/devflow explain-state` 发现控制面和制品 registry 漂移时会给出只读恢复计划;用户审查后可用 `/devflow recover` 确认仅修复可由文件系统证明的 registry 字段,它绝不推进步骤或自动通过 checkpoint。恢复已有流程使用 `/devflow resume`,它同样只恢复状态而不自动执行。
|
|
121
121
|
|
|
122
|
+
### 需求验收测试与 TDD 证据链
|
|
123
|
+
|
|
124
|
+
新建功能在 `integration-verify` 后会先生成独立的 `TC-*` 验收测试设计,再生成变更清单:
|
|
125
|
+
|
|
126
|
+
```text
|
|
127
|
+
已确认需求/契约/架构事实 → TC-* 候选 → 变更清单(任务 ↔ 候选 TC 绑定)
|
|
128
|
+
↓
|
|
129
|
+
CP-3:校验候选绑定 → 冻结 TC → 写入 Core attestation
|
|
130
|
+
↓
|
|
131
|
+
RED / GREEN / REGRESSION 证据 → 按冻结 TC 执行 API / 页面 / 专项测试 → UAT → CP-4
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
- `TC-*` 的步骤和 oracle 在代码出现前冻结;CP-3 在同一次确认中先校验候选绑定,再冻结 TC 并固定绑定哈希。需求或契约变化会使它 stale,不能继续以旧测试集宣称通过。
|
|
135
|
+
- `tdd_required` 任务必须先由 Core 记录准备快照,再记录 RED(命中预期失败)、同一目标的 GREEN 和回归;缺失或环境失败不算完成。
|
|
136
|
+
- API、跨数据域、边界、迁移、工作负载与架构风险会派生相应的 `api`、`permission`、`boundary`、`migration`、`performance`、`resilience` TC 义务。
|
|
137
|
+
- 代码完成后,`workflow-run-acceptance-tests` 只能运行绑定到冻结 TC、且已在初始化 `.planning/toolchain.md` 发现的测试运行器命令。任何失败、未执行、人工待执行或环境阻塞用例都会阻止 CP-4。
|
|
138
|
+
|
|
139
|
+
正常使用不需要记忆底层命令:在 Devflow 工作流中按步骤继续即可。需要排查时,可用 `devflow_acceptance_tests` 查看冻结套件,或由工作流调用 `devflow_tdd_evidence` 与 `devflow_run_acceptance_tests` 生成 Core-owned 证据;不要手写“已测试”来替代结果文件。
|
|
140
|
+
|
|
122
141
|
## Marketplace 能力包
|
|
123
142
|
|
|
124
143
|
仓库也可作为 OMP marketplace 使用,但 Marketplace 安装只用于发现和安装 Skills、Agents、命令等能力内容,不代表完整 Devflow Runtime 已启用。OMP 不会从 Marketplace 安装加载 `package.json` 中的 `omp.extensions`,因此下面的命令不能替代 `bash .skills/install.sh --apply`:
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@guanlin4924/devflow-pi",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.20",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Devflow 专项研发 Pi,基于固定版本 OMP Runtime 组装研发状态机、代码智能、设计、实现与质量门禁。",
|
|
6
6
|
"license": "UNLICENSED",
|
|
@@ -11,8 +11,8 @@
|
|
|
11
11
|
"dev": "bun run src/cli.ts",
|
|
12
12
|
"doctor": "bun run src/cli.ts doctor",
|
|
13
13
|
"test:devflow-pi": "bun test tests/devflow-pi",
|
|
14
|
-
"test": "bun test tests/devflow-pi &&
|
|
15
|
-
"check": "bun test tests/devflow-pi &&
|
|
14
|
+
"test": "bun test tests/devflow-pi && python3 -m unittest discover -s tests -p 'test_*.py'",
|
|
15
|
+
"check": "bun test tests/devflow-pi && python3 -m unittest discover -s tests -p 'test_*.py'"
|
|
16
16
|
},
|
|
17
17
|
"engines": {
|
|
18
18
|
"bun": ">=1.3.14"
|
|
@@ -52,6 +52,7 @@ const READ_ONLY_TOOLS = new Set([
|
|
|
52
52
|
"devflow_doctor",
|
|
53
53
|
"devflow_code_index",
|
|
54
54
|
"devflow_context_cursor",
|
|
55
|
+
"devflow_acceptance_tests",
|
|
55
56
|
"devflow_usage_stats",
|
|
56
57
|
"usage_stats",
|
|
57
58
|
]);
|
|
@@ -68,7 +69,10 @@ const STATE_MUTATION_TOOLS = new Set([
|
|
|
68
69
|
"devflow_bootstrap",
|
|
69
70
|
"devflow_transition",
|
|
70
71
|
"devflow_artifact",
|
|
72
|
+
"devflow_api_test_matrix",
|
|
71
73
|
"devflow_evidence",
|
|
74
|
+
"devflow_tdd_evidence",
|
|
75
|
+
"devflow_run_acceptance_tests",
|
|
72
76
|
]);
|
|
73
77
|
|
|
74
78
|
export function initialControlState(): SessionControlState {
|
|
@@ -335,6 +339,10 @@ const REQUIRED_EXTENSION_TOOLS = [
|
|
|
335
339
|
"devflow_artifact",
|
|
336
340
|
"devflow_machine_evidence",
|
|
337
341
|
"devflow_requirement_graph",
|
|
342
|
+
"devflow_acceptance_tests",
|
|
343
|
+
"devflow_api_test_matrix",
|
|
344
|
+
"devflow_tdd_evidence",
|
|
345
|
+
"devflow_run_acceptance_tests",
|
|
338
346
|
"devflow_evidence",
|
|
339
347
|
"devflow_contract",
|
|
340
348
|
"devflow_recovery",
|
|
@@ -353,6 +361,7 @@ const REQUIRED_SKILLS = [
|
|
|
353
361
|
"workflow-api-call-guide",
|
|
354
362
|
"workflow-api-contract",
|
|
355
363
|
"workflow-api-supplement",
|
|
364
|
+
"workflow-acceptance-test-design",
|
|
356
365
|
"workflow-audit-milestone",
|
|
357
366
|
"workflow-backend-tech-spec",
|
|
358
367
|
"workflow-bug-fix",
|
|
@@ -371,6 +380,7 @@ const REQUIRED_SKILLS = [
|
|
|
371
380
|
"workflow-orchestrator",
|
|
372
381
|
"workflow-pipeline",
|
|
373
382
|
"workflow-project-context",
|
|
383
|
+
"workflow-run-acceptance-tests",
|
|
374
384
|
"workflow-requirement-baseline",
|
|
375
385
|
"workflow-requirement-triage",
|
|
376
386
|
"workflow-ui-spec",
|
|
@@ -408,6 +418,12 @@ const REQUIRED_SCRIPTS = [
|
|
|
408
418
|
"workflow-orchestrator/scripts/advance_state.rb",
|
|
409
419
|
"workflow-orchestrator/scripts/sync_artifacts.rb",
|
|
410
420
|
"workflow-orchestrator/scripts/requirement_graph.rb",
|
|
421
|
+
"workflow-orchestrator/scripts/acceptance_test_suite.rb",
|
|
422
|
+
"workflow-orchestrator/scripts/api_test_matrix.py",
|
|
423
|
+
"workflow-orchestrator/scripts/test_task_binding_check.py",
|
|
424
|
+
"workflow-orchestrator/scripts/test_command_policy.py",
|
|
425
|
+
"workflow-orchestrator/scripts/tdd_evidence.py",
|
|
426
|
+
"workflow-orchestrator/scripts/acceptance_test_runner.py",
|
|
411
427
|
"workflow-orchestrator/scripts/machine_evidence.rb",
|
|
412
428
|
"workflow-orchestrator/scripts/execution_contract.rb",
|
|
413
429
|
"workflow-code-index/scripts/code_index.py",
|
|
@@ -515,7 +531,7 @@ function safeControlPath(cwd: string, raw: string | undefined, kind: "state" | "
|
|
|
515
531
|
}
|
|
516
532
|
|
|
517
533
|
function isCanonicalRequirementGraphPath(filePath: string): boolean {
|
|
518
|
-
return /^\.planning\/requirements\/[^/\\\x00-\x1F\x7F]+\/(?:governance|evidence|definitions|behavior-graph|requirement-graph|slices|acceptance|traceability|graph\.meta)\.json$/u.test(filePath)
|
|
534
|
+
return /^\.planning\/requirements\/[^/\\\x00-\x1F\x7F]+\/(?:governance|evidence|definitions|behavior-graph|requirement-graph|slices|acceptance|tests|traceability|graph\.meta)\.json$/u.test(filePath)
|
|
519
535
|
|| /^\.planning\/requirements\/[^/\\\x00-\x1F\x7F]+\/(?:context\/[^/\\]+\.json|control-events\.jsonl)$/u.test(filePath);
|
|
520
536
|
}
|
|
521
537
|
|
|
@@ -967,6 +983,15 @@ export default function devflowCore(pi: ExtensionAPI) {
|
|
|
967
983
|
});
|
|
968
984
|
}
|
|
969
985
|
|
|
986
|
+
async function runPython(cwd: string, relativeScript: string, args: string[]) {
|
|
987
|
+
assertRuntimeProjectBoundary(cwd);
|
|
988
|
+
const kitRoot = resolveKitRoot(cwd);
|
|
989
|
+
return runProcess(["python3", path.join(kitRoot, relativeScript), ...args], cwd, {
|
|
990
|
+
DEVFLOW_KIT_ROOT: kitRoot,
|
|
991
|
+
OMP_PROJECT_DIR: cwd,
|
|
992
|
+
});
|
|
993
|
+
}
|
|
994
|
+
|
|
970
995
|
async function runHook(
|
|
971
996
|
cwd: string,
|
|
972
997
|
scriptName: string,
|
|
@@ -1520,7 +1545,7 @@ export default function devflowCore(pi: ExtensionAPI) {
|
|
|
1520
1545
|
read: z.boolean().optional(),
|
|
1521
1546
|
delta: z.object({
|
|
1522
1547
|
op: z.enum(["add", "link", "update"]),
|
|
1523
|
-
kind: z.enum(["source", "code", "term", "object", "behavior", "requirement", "rule", "boundary", "slice", "risk", "decision", "acceptance", "api", "workload_shape", "workload_constraint"]).optional(),
|
|
1548
|
+
kind: z.enum(["source", "code", "term", "object", "behavior", "requirement", "rule", "boundary", "slice", "risk", "decision", "acceptance", "api", "workload_shape", "workload_constraint", "test_case"]).optional(),
|
|
1524
1549
|
id: z.string().min(1).optional(),
|
|
1525
1550
|
data: z.record(z.string(), z.unknown()).optional(),
|
|
1526
1551
|
from: z.string().min(1).optional(),
|
|
@@ -1542,6 +1567,14 @@ export default function devflowCore(pi: ExtensionAPI) {
|
|
|
1542
1567
|
if (params.expectedRevision === undefined || !params.delta) {
|
|
1543
1568
|
return renderResult({ ok: false, exitCode: 2, stdout: "", stderr: "propose requires expectedRevision and delta" });
|
|
1544
1569
|
}
|
|
1570
|
+
if (params.delta.kind === "test_case" && activeContract?.step !== "acceptance-test-design") {
|
|
1571
|
+
return renderResult({
|
|
1572
|
+
ok: false,
|
|
1573
|
+
exitCode: 1,
|
|
1574
|
+
stdout: "",
|
|
1575
|
+
stderr: "TC candidates are accepted only during the active acceptance-test-design contract.",
|
|
1576
|
+
});
|
|
1577
|
+
}
|
|
1545
1578
|
args.push("--write", "--expected-revision", String(params.expectedRevision), "--delta-json", JSON.stringify(params.delta));
|
|
1546
1579
|
if (activeContract?.allowedGraphDelta?.length) args.push("--allowed-kinds", activeContract.allowedGraphDelta.join(","));
|
|
1547
1580
|
}
|
|
@@ -1588,6 +1621,113 @@ export default function devflowCore(pi: ExtensionAPI) {
|
|
|
1588
1621
|
},
|
|
1589
1622
|
});
|
|
1590
1623
|
|
|
1624
|
+
pi.registerTool({
|
|
1625
|
+
name: "devflow_acceptance_tests",
|
|
1626
|
+
label: "Devflow Acceptance Tests",
|
|
1627
|
+
description: "Inspect the Core-owned frozen TC acceptance suite. CP-3, not an agent, freezes the suite.",
|
|
1628
|
+
parameters: z.object({
|
|
1629
|
+
feature: z.string().min(1),
|
|
1630
|
+
}),
|
|
1631
|
+
approval: "read",
|
|
1632
|
+
async execute(_id, params, _signal, _onUpdate, ctx) {
|
|
1633
|
+
return renderResult(
|
|
1634
|
+
await runRuby(ctx.cwd, "workflow-orchestrator/scripts/acceptance_test_suite.rb", ["inspect", "--feature", params.feature, "--json"]),
|
|
1635
|
+
{ feature: params.feature },
|
|
1636
|
+
);
|
|
1637
|
+
},
|
|
1638
|
+
});
|
|
1639
|
+
|
|
1640
|
+
pi.registerTool({
|
|
1641
|
+
name: "devflow_api_test_matrix",
|
|
1642
|
+
label: "Devflow API Test Matrix",
|
|
1643
|
+
description: "Compile complete API field-boundary, pairwise and error-code TC candidates; import them only during acceptance-test design.",
|
|
1644
|
+
parameters: z.object({
|
|
1645
|
+
action: z.enum(["compile", "import", "validate"]),
|
|
1646
|
+
feature: z.string().min(1),
|
|
1647
|
+
}),
|
|
1648
|
+
approval: args => args.action === "validate" ? "read" : "write",
|
|
1649
|
+
async execute(_id, params, _signal, _onUpdate, ctx) {
|
|
1650
|
+
if (params.action !== "validate" && activeContract?.step !== "acceptance-test-design") {
|
|
1651
|
+
return renderResult({ ok: false, exitCode: 1, stdout: "", stderr: "API test matrix compilation/import is allowed only during the active acceptance-test-design contract." });
|
|
1652
|
+
}
|
|
1653
|
+
if (params.action === "validate") {
|
|
1654
|
+
const process = await runPython(ctx.cwd, "workflow-orchestrator/scripts/api_test_matrix.py", ["validate", "--feature", params.feature, "--json"]);
|
|
1655
|
+
const payload = parseJsonResult(process);
|
|
1656
|
+
const failed = payload && typeof payload === "object" && (payload as { ok?: unknown }).ok === false;
|
|
1657
|
+
return renderResult(failed ? { ...process, ok: false, exitCode: process.exitCode || 1 } : process, { action: params.action, feature: params.feature });
|
|
1658
|
+
}
|
|
1659
|
+
const compiled = await runPython(ctx.cwd, "workflow-orchestrator/scripts/api_test_matrix.py", ["compile", "--feature", params.feature, "--write", "--json"]);
|
|
1660
|
+
const compiledPayload = parseJsonResult(compiled);
|
|
1661
|
+
if (!compiled.ok || !compiledPayload || typeof compiledPayload !== "object" || (compiledPayload as { ok?: unknown }).ok !== true) {
|
|
1662
|
+
return renderResult({ ...compiled, ok: false, exitCode: compiled.exitCode || 1 }, { action: params.action, feature: params.feature });
|
|
1663
|
+
}
|
|
1664
|
+
if (params.action === "compile") return renderResult(compiled, { action: params.action, feature: params.feature });
|
|
1665
|
+
|
|
1666
|
+
const inspected = parseJsonResult(await runRuby(ctx.cwd, "workflow-orchestrator/scripts/requirement_graph.rb", ["inspect", "--feature", params.feature, "--json"]));
|
|
1667
|
+
const revision = inspected && typeof inspected === "object" && typeof (inspected as { graph?: { revision?: unknown } }).graph?.revision === "number"
|
|
1668
|
+
? (inspected as { graph: { revision: number } }).graph.revision
|
|
1669
|
+
: undefined;
|
|
1670
|
+
if (revision === undefined) return renderResult({ ok: false, exitCode: 1, stdout: "", stderr: "Could not read the current requirement-graph revision before importing API TC candidates." });
|
|
1671
|
+
const imported = await runRuby(ctx.cwd, "workflow-orchestrator/scripts/requirement_graph.rb", ["import-test-cases", "--feature", params.feature, "--write", "--expected-revision", String(revision), "--json"]);
|
|
1672
|
+
const importedPayload = parseJsonResult(imported);
|
|
1673
|
+
const failed = importedPayload && typeof importedPayload === "object" && (importedPayload as { ok?: unknown }).ok === false;
|
|
1674
|
+
return renderResult(failed ? { ...imported, ok: false, exitCode: imported.exitCode || 1 } : imported, { action: params.action, feature: params.feature });
|
|
1675
|
+
},
|
|
1676
|
+
});
|
|
1677
|
+
|
|
1678
|
+
pi.registerTool({
|
|
1679
|
+
name: "devflow_tdd_evidence",
|
|
1680
|
+
label: "Devflow TDD Evidence",
|
|
1681
|
+
description: "Core-owned preparation and RED/GREEN/REGRESSION evidence for a frozen tdd_required task binding.",
|
|
1682
|
+
parameters: z.object({
|
|
1683
|
+
action: z.enum(["prepare", "record", "inspect"]),
|
|
1684
|
+
feature: z.string().min(1),
|
|
1685
|
+
taskId: z.string().min(1),
|
|
1686
|
+
stage: z.enum(["red", "green", "regression"]).optional(),
|
|
1687
|
+
}),
|
|
1688
|
+
approval: args => args.action === "inspect" ? "read" : "write",
|
|
1689
|
+
async execute(_id, params, _signal, _onUpdate, ctx) {
|
|
1690
|
+
if (params.action === "record" && !params.stage) {
|
|
1691
|
+
return renderResult({ ok: false, exitCode: 2, stdout: "", stderr: "record requires stage" });
|
|
1692
|
+
}
|
|
1693
|
+
const args = [params.action, "--feature", params.feature, "--task-id", params.taskId, "--json"];
|
|
1694
|
+
if (params.action !== "inspect") args.push("--write");
|
|
1695
|
+
if (params.stage) args.push("--stage", params.stage);
|
|
1696
|
+
const process = await runPython(ctx.cwd, "workflow-orchestrator/scripts/tdd_evidence.py", args);
|
|
1697
|
+
const payload = parseJsonResult(process);
|
|
1698
|
+
const failed = payload && typeof payload === "object" && (payload as { ok?: unknown }).ok === false;
|
|
1699
|
+
return renderResult(failed ? { ...process, ok: false, exitCode: process.exitCode || 1 } : process, {
|
|
1700
|
+
action: params.action,
|
|
1701
|
+
feature: params.feature,
|
|
1702
|
+
taskId: params.taskId,
|
|
1703
|
+
stage: params.stage,
|
|
1704
|
+
});
|
|
1705
|
+
},
|
|
1706
|
+
});
|
|
1707
|
+
|
|
1708
|
+
pi.registerTool({
|
|
1709
|
+
name: "devflow_run_acceptance_tests",
|
|
1710
|
+
label: "Devflow Run Acceptance Tests",
|
|
1711
|
+
description: "Validate or execute only command or portable HTTP adapters bound to the frozen TC acceptance suite.",
|
|
1712
|
+
parameters: z.object({
|
|
1713
|
+
action: z.enum(["validate", "run", "validate_results"]),
|
|
1714
|
+
feature: z.string().min(1),
|
|
1715
|
+
}),
|
|
1716
|
+
approval: args => args.action === "run" ? "write" : "read",
|
|
1717
|
+
async execute(_id, params, _signal, _onUpdate, ctx) {
|
|
1718
|
+
const scriptAction = params.action === "validate_results" ? "validate-results" : params.action;
|
|
1719
|
+
const args = [scriptAction, "--feature", params.feature, "--json"];
|
|
1720
|
+
if (params.action === "run") args.push("--write");
|
|
1721
|
+
const process = await runPython(ctx.cwd, "workflow-orchestrator/scripts/acceptance_test_runner.py", args);
|
|
1722
|
+
const payload = parseJsonResult(process);
|
|
1723
|
+
const failed = payload && typeof payload === "object" && (payload as { ok?: unknown }).ok === false;
|
|
1724
|
+
return renderResult(failed ? { ...process, ok: false, exitCode: process.exitCode || 1 } : process, {
|
|
1725
|
+
action: params.action,
|
|
1726
|
+
feature: params.feature,
|
|
1727
|
+
});
|
|
1728
|
+
},
|
|
1729
|
+
});
|
|
1730
|
+
|
|
1591
1731
|
pi.registerTool({
|
|
1592
1732
|
name: "devflow_runtime_context",
|
|
1593
1733
|
label: "Devflow Runtime Context",
|
|
@@ -0,0 +1,394 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Execute only adapters bound to a frozen TC suite; never derive scope from delivery prose."""
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import hashlib
|
|
7
|
+
import json
|
|
8
|
+
import os
|
|
9
|
+
import re
|
|
10
|
+
import subprocess
|
|
11
|
+
from datetime import datetime, timezone
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from typing import Any
|
|
14
|
+
from urllib.error import HTTPError, URLError
|
|
15
|
+
from urllib.parse import urlencode, urljoin
|
|
16
|
+
from urllib.request import Request, urlopen
|
|
17
|
+
|
|
18
|
+
from test_command_policy import validate_test_command
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
FEATURE_RE = re.compile(r"^[^/\\\x00-\x1f\x7f]+$")
|
|
22
|
+
AUTOMATION_MODES = {"automated", "manual", "blocked"}
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def utc_now() -> str:
|
|
26
|
+
return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def digest(value: str | bytes) -> str:
|
|
30
|
+
if isinstance(value, str):
|
|
31
|
+
value = value.encode("utf-8")
|
|
32
|
+
return hashlib.sha256(value).hexdigest()
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def read_json(path: Path, errors: list[str], label: str) -> dict[str, Any] | None:
|
|
36
|
+
try:
|
|
37
|
+
value = json.loads(path.read_text(encoding="utf-8"))
|
|
38
|
+
except FileNotFoundError:
|
|
39
|
+
errors.append(f"missing {label}: {path.as_posix()}")
|
|
40
|
+
return None
|
|
41
|
+
except json.JSONDecodeError as error:
|
|
42
|
+
errors.append(f"invalid {label}: {error.msg}")
|
|
43
|
+
return None
|
|
44
|
+
if not isinstance(value, dict):
|
|
45
|
+
errors.append(f"invalid {label}: root must be an object")
|
|
46
|
+
return None
|
|
47
|
+
return value
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def write_json(path: Path, value: dict[str, Any]) -> None:
|
|
51
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
52
|
+
temporary = path.with_name(f".{path.name}.tmp")
|
|
53
|
+
temporary.write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
54
|
+
temporary.replace(path)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def valid_argv(value: Any) -> bool:
|
|
58
|
+
return isinstance(value, list) and bool(value) and all(isinstance(token, str) and token and "\x00" not in token for token in value)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
ENV_NAME_RE = re.compile(r"^[A-Z_][A-Z0-9_]*$")
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def valid_http_adapter(value: Any) -> bool:
|
|
65
|
+
if not isinstance(value, dict) or value.get("kind") != "http" or not isinstance(value.get("base_url_env"), str) or not ENV_NAME_RE.fullmatch(value["base_url_env"]):
|
|
66
|
+
return False
|
|
67
|
+
headers_env = value.get("headers_env", {})
|
|
68
|
+
if not isinstance(headers_env, dict) or not all(isinstance(header, str) and header and isinstance(env, str) and ENV_NAME_RE.fullmatch(env) for header, env in headers_env.items()):
|
|
69
|
+
return False
|
|
70
|
+
setup = value.get("setup")
|
|
71
|
+
return setup is None or (isinstance(setup, dict) and valid_argv(setup.get("argv")))
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def pointer_value(document: Any, pointer: str) -> Any:
|
|
75
|
+
if pointer == "":
|
|
76
|
+
return document
|
|
77
|
+
if not isinstance(pointer, str) or not pointer.startswith("/"):
|
|
78
|
+
raise ValueError("error_code_pointer must be an RFC 6901 JSON pointer")
|
|
79
|
+
current = document
|
|
80
|
+
for token in pointer[1:].split("/"):
|
|
81
|
+
token = token.replace("~1", "/").replace("~0", "~")
|
|
82
|
+
if isinstance(current, dict) and token in current:
|
|
83
|
+
current = current[token]
|
|
84
|
+
elif isinstance(current, list) and token.isdigit() and int(token) < len(current):
|
|
85
|
+
current = current[int(token)]
|
|
86
|
+
else:
|
|
87
|
+
raise KeyError(pointer)
|
|
88
|
+
return current
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def http_execution_error(source: dict[str, Any]) -> str | None:
|
|
92
|
+
execution = source.get("execution")
|
|
93
|
+
if not isinstance(execution, dict) or execution.get("protocol") != "http":
|
|
94
|
+
return "frozen TC has no HTTP execution definition"
|
|
95
|
+
request = execution.get("request")
|
|
96
|
+
expect = execution.get("expect")
|
|
97
|
+
if not isinstance(request, dict) or not isinstance(expect, dict):
|
|
98
|
+
return "frozen TC HTTP execution requires request and expect"
|
|
99
|
+
if not isinstance(request.get("method"), str) or not re.fullmatch(r"[A-Z]+", request["method"]):
|
|
100
|
+
return "frozen TC HTTP request method is invalid"
|
|
101
|
+
if not isinstance(request.get("path"), str) or not request["path"].startswith("/"):
|
|
102
|
+
return "frozen TC HTTP request path is invalid"
|
|
103
|
+
if not isinstance(expect.get("http_status"), int) or not 100 <= expect["http_status"] <= 599:
|
|
104
|
+
return "frozen TC HTTP expectation http_status is invalid"
|
|
105
|
+
for key in ("path_params", "query", "headers", "body"):
|
|
106
|
+
if key in request and not isinstance(request[key], dict):
|
|
107
|
+
return f"frozen TC HTTP request {key} must be an object"
|
|
108
|
+
if "error_code" in expect and (not isinstance(expect["error_code"], str) or not expect["error_code"].strip()):
|
|
109
|
+
return "frozen TC HTTP expectation error_code is invalid"
|
|
110
|
+
if "error_code_pointer" in expect and (not isinstance(expect["error_code_pointer"], str) or not expect["error_code_pointer"].startswith("/")):
|
|
111
|
+
return "frozen TC HTTP expectation error_code_pointer is invalid"
|
|
112
|
+
return None
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def frozen_cases(suite: dict[str, Any], errors: list[str]) -> dict[str, dict[str, Any]]:
|
|
116
|
+
cases: dict[str, dict[str, Any]] = {}
|
|
117
|
+
for node in suite.get("nodes", []):
|
|
118
|
+
if not isinstance(node, dict) or node.get("kind") != "test_case":
|
|
119
|
+
continue
|
|
120
|
+
tc_id = node.get("id")
|
|
121
|
+
data = node.get("data")
|
|
122
|
+
if not isinstance(tc_id, str) or not isinstance(data, dict):
|
|
123
|
+
errors.append("invalid test_case node in TC suite")
|
|
124
|
+
continue
|
|
125
|
+
if node.get("status") != "confirmed" or node.get("stale") is True:
|
|
126
|
+
errors.append(f"TC is not frozen and fresh: {tc_id}")
|
|
127
|
+
cases[tc_id] = data
|
|
128
|
+
if not cases:
|
|
129
|
+
errors.append("TC suite contains no test_case nodes")
|
|
130
|
+
return cases
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def paths(root: Path, feature: str) -> tuple[Path, Path, Path]:
|
|
134
|
+
return (
|
|
135
|
+
root / ".planning" / "requirements" / feature / "tests.json",
|
|
136
|
+
root / ".planning" / "features" / feature / "acceptance-test-bindings.json",
|
|
137
|
+
root / ".planning" / "test-runs" / feature / "acceptance-results.json",
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def file_fingerprint(path: Path) -> str | None:
|
|
142
|
+
return digest(path.read_bytes()) if path.is_file() else None
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def run_http_case(source: dict[str, Any], adapter: dict[str, Any]) -> tuple[bool, int, str]:
|
|
146
|
+
execution = source["execution"]
|
|
147
|
+
request_spec = execution["request"]
|
|
148
|
+
expect = execution["expect"]
|
|
149
|
+
base_url = os.environ.get(adapter["base_url_env"])
|
|
150
|
+
if not base_url:
|
|
151
|
+
return False, 127, f"required test base URL environment variable is unset: {adapter['base_url_env']}"
|
|
152
|
+
setup_output = ""
|
|
153
|
+
setup = adapter.get("setup")
|
|
154
|
+
if isinstance(setup, dict):
|
|
155
|
+
try:
|
|
156
|
+
completed = subprocess.run(setup["argv"], text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=180, check=False)
|
|
157
|
+
setup_output = f"setup exit={completed.returncode}: {completed.stdout}\n{completed.stderr}"
|
|
158
|
+
if completed.returncode != 0:
|
|
159
|
+
return False, completed.returncode, setup_output
|
|
160
|
+
except subprocess.TimeoutExpired as error:
|
|
161
|
+
return False, 124, f"setup command timed out: {error}"
|
|
162
|
+
except OSError as error:
|
|
163
|
+
return False, 127, f"setup command could not start: {error}"
|
|
164
|
+
path = request_spec["path"]
|
|
165
|
+
for key, value in request_spec.get("path_params", {}).items():
|
|
166
|
+
path = path.replace("{" + str(key) + "}", str(value)).replace(":" + str(key), str(value))
|
|
167
|
+
query = request_spec.get("query", {})
|
|
168
|
+
url = urljoin(base_url.rstrip("/") + "/", path.lstrip("/"))
|
|
169
|
+
if query:
|
|
170
|
+
url += ("&" if "?" in url else "?") + urlencode(query, doseq=True)
|
|
171
|
+
headers = {str(key): str(value) for key, value in request_spec.get("headers", {}).items()}
|
|
172
|
+
for header, env_name in adapter.get("headers_env", {}).items():
|
|
173
|
+
value = os.environ.get(env_name)
|
|
174
|
+
if not value:
|
|
175
|
+
return False, 127, f"required test header environment variable is unset: {env_name}"
|
|
176
|
+
headers[header] = value
|
|
177
|
+
body = request_spec.get("body")
|
|
178
|
+
payload: bytes | None = None
|
|
179
|
+
if body is not None:
|
|
180
|
+
payload = json.dumps(body, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
|
|
181
|
+
headers.setdefault("Content-Type", "application/json")
|
|
182
|
+
request = Request(url, data=payload, headers=headers, method=request_spec["method"])
|
|
183
|
+
try:
|
|
184
|
+
with urlopen(request, timeout=20) as response:
|
|
185
|
+
http_status = response.status
|
|
186
|
+
output = response.read().decode("utf-8", errors="replace")
|
|
187
|
+
except HTTPError as error:
|
|
188
|
+
http_status = error.code
|
|
189
|
+
output = error.read().decode("utf-8", errors="replace")
|
|
190
|
+
except (URLError, OSError) as error:
|
|
191
|
+
return False, 127, f"{setup_output}\nHTTP adapter could not complete request: {error}"
|
|
192
|
+
expected_status = expect["http_status"]
|
|
193
|
+
if http_status != expected_status:
|
|
194
|
+
return False, 1, f"{setup_output}\nexpected HTTP {expected_status}, got {http_status}; body={output[:1000]}"
|
|
195
|
+
if "error_code" in expect:
|
|
196
|
+
try:
|
|
197
|
+
body_json = json.loads(output)
|
|
198
|
+
actual_code = pointer_value(body_json, expect.get("error_code_pointer", "/error/code"))
|
|
199
|
+
except (json.JSONDecodeError, KeyError, ValueError) as error:
|
|
200
|
+
return False, 1, f"{setup_output}\ncould not read expected error code from response: {error}; body={output[:1000]}"
|
|
201
|
+
if actual_code != expect["error_code"]:
|
|
202
|
+
return False, 1, f"{setup_output}\nexpected error code {expect['error_code']}, got {actual_code!r}; body={output[:1000]}"
|
|
203
|
+
return True, 0, f"{setup_output}\nHTTP {http_status} matched expectation"
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def validate(root: Path, feature: str) -> tuple[dict[str, Any], dict[str, Any] | None, dict[str, Any] | None]:
|
|
207
|
+
errors: list[str] = []
|
|
208
|
+
suite_path, binding_path, _ = paths(root, feature)
|
|
209
|
+
suite = read_json(suite_path, errors, "TC suite")
|
|
210
|
+
bindings = read_json(binding_path, errors, "acceptance-test bindings")
|
|
211
|
+
if suite is None or bindings is None:
|
|
212
|
+
return {"ok": False, "feature": feature, "errors": errors}, suite, bindings
|
|
213
|
+
cases = frozen_cases(suite, errors)
|
|
214
|
+
if bindings.get("schema_version") != 1:
|
|
215
|
+
errors.append("acceptance-test bindings schema_version must be 1")
|
|
216
|
+
if bindings.get("origin") != "devflow_acceptance_test_execution":
|
|
217
|
+
errors.append("acceptance-test bindings origin must be devflow_acceptance_test_execution")
|
|
218
|
+
if bindings.get("feature") != feature:
|
|
219
|
+
errors.append("acceptance-test bindings feature does not match requested feature")
|
|
220
|
+
pin = bindings.get("test_suite")
|
|
221
|
+
meta = suite.get("view_meta") if isinstance(suite.get("view_meta"), dict) else {}
|
|
222
|
+
if not isinstance(pin, dict):
|
|
223
|
+
errors.append("test_suite pin is required")
|
|
224
|
+
else:
|
|
225
|
+
if pin.get("graph_revision") != suite.get("graph_revision"):
|
|
226
|
+
errors.append("test_suite.graph_revision does not match current TC suite")
|
|
227
|
+
if pin.get("fingerprint") != meta.get("fingerprint"):
|
|
228
|
+
errors.append("test_suite.fingerprint does not match current TC suite")
|
|
229
|
+
|
|
230
|
+
entries = bindings.get("cases")
|
|
231
|
+
if not isinstance(entries, list) or not entries:
|
|
232
|
+
errors.append("cases must be a non-empty array")
|
|
233
|
+
entries = []
|
|
234
|
+
seen: set[str] = set()
|
|
235
|
+
for index, entry in enumerate(entries):
|
|
236
|
+
label = f"cases[{index}]"
|
|
237
|
+
if not isinstance(entry, dict):
|
|
238
|
+
errors.append(f"{label} must be an object")
|
|
239
|
+
continue
|
|
240
|
+
tc_id = entry.get("tc_id")
|
|
241
|
+
if not isinstance(tc_id, str) or not tc_id:
|
|
242
|
+
errors.append(f"{label}.tc_id is required")
|
|
243
|
+
continue
|
|
244
|
+
if tc_id in seen:
|
|
245
|
+
errors.append(f"duplicate acceptance-test binding: {tc_id}")
|
|
246
|
+
seen.add(tc_id)
|
|
247
|
+
source = cases.get(tc_id)
|
|
248
|
+
if source is None:
|
|
249
|
+
errors.append(f"unknown TC reference: {tc_id}")
|
|
250
|
+
continue
|
|
251
|
+
mode = entry.get("mode")
|
|
252
|
+
if mode not in AUTOMATION_MODES:
|
|
253
|
+
errors.append(f"{label}.mode is invalid")
|
|
254
|
+
continue
|
|
255
|
+
if mode != source.get("automation_mode"):
|
|
256
|
+
errors.append(f"{label}.mode must match frozen TC automation_mode")
|
|
257
|
+
if mode == "automated":
|
|
258
|
+
adapter = entry.get("adapter")
|
|
259
|
+
if not isinstance(adapter, dict):
|
|
260
|
+
errors.append(f"{label}.adapter is required")
|
|
261
|
+
elif adapter.get("kind") == "command":
|
|
262
|
+
if not valid_argv(adapter.get("argv")):
|
|
263
|
+
errors.append(f"{label}.adapter command requires non-empty argv")
|
|
264
|
+
continue
|
|
265
|
+
command_error = validate_test_command(root, adapter["argv"])
|
|
266
|
+
if command_error:
|
|
267
|
+
errors.append(f"{label}.adapter: {command_error}")
|
|
268
|
+
elif adapter.get("kind") == "http":
|
|
269
|
+
if not valid_http_adapter(adapter):
|
|
270
|
+
errors.append(f"{label}.adapter HTTP requires base_url_env, valid optional headers_env, and optional setup.argv")
|
|
271
|
+
else:
|
|
272
|
+
execution_error = http_execution_error(source)
|
|
273
|
+
if execution_error:
|
|
274
|
+
errors.append(f"{label}.adapter: {execution_error}")
|
|
275
|
+
if isinstance(adapter.get("setup"), dict):
|
|
276
|
+
setup_error = validate_test_command(root, adapter["setup"]["argv"])
|
|
277
|
+
if setup_error:
|
|
278
|
+
errors.append(f"{label}.adapter.setup: {setup_error}")
|
|
279
|
+
else:
|
|
280
|
+
errors.append(f"{label}.adapter kind must be command or http")
|
|
281
|
+
elif mode == "manual":
|
|
282
|
+
if not isinstance(entry.get("owner"), str) or not entry["owner"].strip() or not isinstance(entry.get("reason"), str) or not entry["reason"].strip():
|
|
283
|
+
errors.append(f"{label}.manual requires owner and reason")
|
|
284
|
+
elif mode == "blocked":
|
|
285
|
+
if not isinstance(entry.get("blocked_by"), str) or not entry["blocked_by"].strip() or not isinstance(entry.get("owner"), str) or not entry["owner"].strip():
|
|
286
|
+
errors.append(f"{label}.blocked requires blocked_by and owner")
|
|
287
|
+
for tc_id in sorted(set(cases) - seen):
|
|
288
|
+
errors.append(f"unbound frozen TC: {tc_id}")
|
|
289
|
+
return {"ok": not errors, "feature": feature, "tc_ids": sorted(cases), "errors": errors}, suite, bindings
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
def run(root: Path, feature: str, write: bool) -> dict[str, Any]:
|
|
293
|
+
validation, suite, bindings = validate(root, feature)
|
|
294
|
+
if not validation["ok"]:
|
|
295
|
+
return validation
|
|
296
|
+
assert suite is not None and bindings is not None
|
|
297
|
+
cases = frozen_cases(suite, [])
|
|
298
|
+
_, _, result_path = paths(root, feature)
|
|
299
|
+
previous = read_json(result_path, [], "previous acceptance-test results")
|
|
300
|
+
task_bindings_path = root / ".planning" / "features" / feature / "test-bindings.json"
|
|
301
|
+
execution_bindings_path = root / ".planning" / "features" / feature / "acceptance-test-bindings.json"
|
|
302
|
+
results: list[dict[str, Any]] = []
|
|
303
|
+
for entry in bindings["cases"]:
|
|
304
|
+
tc_id = entry["tc_id"]
|
|
305
|
+
mode = entry["mode"]
|
|
306
|
+
if mode == "automated":
|
|
307
|
+
started_at = utc_now()
|
|
308
|
+
adapter = entry["adapter"]
|
|
309
|
+
if adapter["kind"] == "command":
|
|
310
|
+
argv = adapter["argv"]
|
|
311
|
+
try:
|
|
312
|
+
completed = subprocess.run(argv, cwd=root, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=180, check=False)
|
|
313
|
+
exit_code = completed.returncode
|
|
314
|
+
output = f"{completed.stdout}\n{completed.stderr}"
|
|
315
|
+
except subprocess.TimeoutExpired as error:
|
|
316
|
+
exit_code = 124
|
|
317
|
+
output = f"command timed out: {error}"
|
|
318
|
+
except OSError as error:
|
|
319
|
+
exit_code = 127
|
|
320
|
+
output = f"command could not start: {error}"
|
|
321
|
+
result = {"tc_id": tc_id, "status": "passed" if exit_code == 0 else "failed", "mode": mode, "adapter_kind": "command", "argv": argv,
|
|
322
|
+
"command_hash": digest(json.dumps(argv, ensure_ascii=False, separators=(",", ":"))), "started_at": started_at,
|
|
323
|
+
"finished_at": utc_now(), "exit_code": exit_code, "output_sha256": digest(output)}
|
|
324
|
+
else:
|
|
325
|
+
passed, exit_code, output = run_http_case(cases[tc_id], adapter)
|
|
326
|
+
result = {"tc_id": tc_id, "status": "passed" if passed else "failed", "mode": mode, "adapter_kind": "http",
|
|
327
|
+
"adapter_fingerprint": digest(json.dumps(adapter, ensure_ascii=False, separators=(",", ":"))), "started_at": started_at,
|
|
328
|
+
"finished_at": utc_now(), "exit_code": exit_code, "output_sha256": digest(output)}
|
|
329
|
+
results.append(result)
|
|
330
|
+
elif mode == "manual":
|
|
331
|
+
results.append({"tc_id": tc_id, "status": "blocked", "mode": mode, "owner": entry["owner"], "reason": entry["reason"],
|
|
332
|
+
"blocked_by": "manual execution has no Core-observed result"})
|
|
333
|
+
else:
|
|
334
|
+
results.append({"tc_id": tc_id, "status": "blocked", "mode": mode, "owner": entry["owner"], "blocked_by": entry["blocked_by"]})
|
|
335
|
+
all_passed = all(item["status"] == "passed" for item in results)
|
|
336
|
+
history = previous.get("history", []) if isinstance(previous, dict) and isinstance(previous.get("history"), list) else []
|
|
337
|
+
if isinstance(previous, dict) and isinstance(previous.get("results"), list):
|
|
338
|
+
history = [*history, {"executed_at": previous.get("executed_at"), "passed": previous.get("passed"), "test_suite": previous.get("test_suite"),
|
|
339
|
+
"implementation_task_bindings": previous.get("implementation_task_bindings"), "execution_binding_fingerprint": previous.get("execution_binding_fingerprint"),
|
|
340
|
+
"results": previous["results"]}]
|
|
341
|
+
payload = {"schema_version": 1, "origin": "core_acceptance_test_runner", "feature": feature, "executed_at": utc_now(),
|
|
342
|
+
"test_suite": bindings["test_suite"], "implementation_task_bindings": {"path": str(task_bindings_path.relative_to(root)), "fingerprint": file_fingerprint(task_bindings_path)},
|
|
343
|
+
"execution_binding_fingerprint": file_fingerprint(execution_bindings_path), "results": results, "passed": all_passed, "history": history[-20:]}
|
|
344
|
+
if write:
|
|
345
|
+
write_json(result_path, payload)
|
|
346
|
+
return {"ok": all_passed, "feature": feature, "path": str(result_path.relative_to(root)), "results": results,
|
|
347
|
+
"errors": [] if all_passed else ["one or more frozen TC cases failed or are blocked"]}
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
def validate_results(root: Path, feature: str) -> dict[str, Any]:
|
|
351
|
+
validation, suite, _ = validate(root, feature)
|
|
352
|
+
errors = list(validation["errors"])
|
|
353
|
+
_, _, result_path = paths(root, feature)
|
|
354
|
+
results = read_json(result_path, errors, "acceptance-test results")
|
|
355
|
+
if suite is None or results is None:
|
|
356
|
+
return {"ok": False, "feature": feature, "errors": errors}
|
|
357
|
+
if results.get("origin") != "core_acceptance_test_runner" or results.get("feature") != feature:
|
|
358
|
+
errors.append("acceptance-test results identity is invalid")
|
|
359
|
+
meta = suite.get("view_meta") if isinstance(suite.get("view_meta"), dict) else {}
|
|
360
|
+
if results.get("test_suite") != {"graph_revision": suite.get("graph_revision"), "fingerprint": meta.get("fingerprint")}:
|
|
361
|
+
errors.append("acceptance-test results are pinned to a stale TC suite")
|
|
362
|
+
expected = set(validation.get("tc_ids", []))
|
|
363
|
+
result_entries = results.get("results") if isinstance(results.get("results"), list) else []
|
|
364
|
+
actual = {item.get("tc_id") for item in result_entries if isinstance(item, dict) and isinstance(item.get("tc_id"), str)}
|
|
365
|
+
if actual != expected:
|
|
366
|
+
errors.append("acceptance-test results do not cover exactly the frozen TC suite")
|
|
367
|
+
failed = sorted(item.get("tc_id", "<unknown>") for item in result_entries if not isinstance(item, dict) or item.get("status") != "passed")
|
|
368
|
+
if failed:
|
|
369
|
+
errors.append(f"frozen TC cases not passed: {', '.join(failed)}")
|
|
370
|
+
return {"ok": not errors, "feature": feature, "errors": errors, "result_path": str(result_path.relative_to(root))}
|
|
371
|
+
|
|
372
|
+
|
|
373
|
+
def main() -> int:
|
|
374
|
+
parser = argparse.ArgumentParser()
|
|
375
|
+
parser.add_argument("action", choices=("validate", "run", "validate-results"))
|
|
376
|
+
parser.add_argument("--feature", required=True)
|
|
377
|
+
parser.add_argument("--write", action="store_true")
|
|
378
|
+
parser.add_argument("--json", action="store_true")
|
|
379
|
+
args = parser.parse_args()
|
|
380
|
+
root = Path.cwd().resolve()
|
|
381
|
+
if not FEATURE_RE.fullmatch(args.feature):
|
|
382
|
+
result = {"ok": False, "errors": ["invalid feature"]}
|
|
383
|
+
elif args.action == "validate":
|
|
384
|
+
result, _, _ = validate(root, args.feature)
|
|
385
|
+
elif args.action == "run":
|
|
386
|
+
result = run(root, args.feature, args.write) if args.write else {"ok": False, "errors": ["--write is required for run"]}
|
|
387
|
+
else:
|
|
388
|
+
result = validate_results(root, args.feature)
|
|
389
|
+
print(json.dumps(result, ensure_ascii=False, indent=2))
|
|
390
|
+
return 0
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
if __name__ == "__main__":
|
|
394
|
+
raise SystemExit(main())
|