@guanlin4924/devflow-pi 0.1.94 → 0.1.95
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -4
- package/node_modules/@playwright/test/LICENSE +202 -0
- package/node_modules/@playwright/test/NOTICE +5 -0
- package/node_modules/@playwright/test/README.md +318 -0
- package/node_modules/@playwright/test/cli.js +19 -0
- package/node_modules/@playwright/test/index.d.ts +18 -0
- package/node_modules/@playwright/test/index.js +17 -0
- package/node_modules/@playwright/test/index.mjs +18 -0
- package/node_modules/@playwright/test/package.json +35 -0
- package/node_modules/@playwright/test/reporter.d.ts +17 -0
- package/node_modules/@playwright/test/reporter.js +17 -0
- package/node_modules/@playwright/test/reporter.mjs +17 -0
- package/node_modules/playwright/LICENSE +202 -0
- package/node_modules/playwright/NOTICE +5 -0
- package/node_modules/playwright/README.md +318 -0
- package/node_modules/playwright/ThirdPartyNotices.txt +14 -0
- package/node_modules/playwright/cli.js +19 -0
- package/node_modules/playwright/index.d.ts +17 -0
- package/node_modules/playwright/index.js +17 -0
- package/node_modules/playwright/index.mjs +18 -0
- package/node_modules/playwright/jsx-runtime.js +42 -0
- package/node_modules/playwright/jsx-runtime.mjs +21 -0
- package/node_modules/playwright/lib/agents/agentParser.js +89 -0
- package/node_modules/playwright/lib/agents/copilot-setup-steps.yml +34 -0
- package/node_modules/playwright/lib/agents/generateAgents.js +395 -0
- package/node_modules/playwright/lib/agents/playwright-test-coverage.prompt.md +31 -0
- package/node_modules/playwright/lib/agents/playwright-test-generate.prompt.md +8 -0
- package/node_modules/playwright/lib/agents/playwright-test-generator.agent.md +88 -0
- package/node_modules/playwright/lib/agents/playwright-test-heal.prompt.md +6 -0
- package/node_modules/playwright/lib/agents/playwright-test-healer.agent.md +56 -0
- package/node_modules/playwright/lib/agents/playwright-test-plan.prompt.md +9 -0
- package/node_modules/playwright/lib/agents/playwright-test-planner.agent.md +74 -0
- package/node_modules/playwright/lib/cli/reportActions.js +78 -0
- package/node_modules/playwright/lib/cli/testActions.js +214 -0
- package/node_modules/playwright/lib/common/index.js +2923 -0
- package/node_modules/playwright/lib/common/index.js.txt +34 -0
- package/node_modules/playwright/lib/errorContext.js +129 -0
- package/node_modules/playwright/lib/globals.js +58 -0
- package/node_modules/playwright/lib/index.js +815 -0
- package/node_modules/playwright/lib/isomorphic.js +260 -0
- package/node_modules/playwright/lib/isomorphic.js.txt +9 -0
- package/node_modules/playwright/lib/loader/loaderProcessEntry.js +34 -0
- package/node_modules/playwright/lib/loader/loaderProcessEntry.js.txt +9 -0
- package/node_modules/playwright/lib/matchers/expect.js +13490 -0
- package/node_modules/playwright/lib/matchers/expect.js.LICENSE +640 -0
- package/node_modules/playwright/lib/matchers/expect.js.txt +67 -0
- package/node_modules/playwright/lib/mcp/test/browserBackend.js +135 -0
- package/node_modules/playwright/lib/mcp/test/generatorTools.js +122 -0
- package/node_modules/playwright/lib/mcp/test/plannerTools.js +150 -0
- package/node_modules/playwright/lib/mcp/test/seed.js +82 -0
- package/node_modules/playwright/lib/mcp/test/streams.js +44 -0
- package/node_modules/playwright/lib/mcp/test/testBackend.js +99 -0
- package/node_modules/playwright/lib/mcp/test/testContext.js +306 -0
- package/node_modules/playwright/lib/mcp/test/testTool.js +30 -0
- package/node_modules/playwright/lib/mcp/test/testTools.js +98 -0
- package/node_modules/playwright/lib/package.js +47 -0
- package/node_modules/playwright/lib/program.js +241 -0
- package/node_modules/playwright/lib/runner/index.js +8527 -0
- package/node_modules/playwright/lib/runner/index.js.txt +61 -0
- package/node_modules/playwright/lib/transform/babelBundle.js +72274 -0
- package/node_modules/playwright/lib/transform/babelBundle.js.LICENSE +2384 -0
- package/node_modules/playwright/lib/transform/babelBundle.js.txt +325 -0
- package/node_modules/playwright/lib/transform/esmLoader.js +8054 -0
- package/node_modules/playwright/lib/transform/esmLoader.js.LICENSE +355 -0
- package/node_modules/playwright/lib/transform/esmLoader.js.txt +60 -0
- package/node_modules/playwright/lib/util.js +373 -0
- package/node_modules/playwright/lib/worker/workerProcessEntry.js +1861 -0
- package/node_modules/playwright/lib/worker/workerProcessEntry.js.txt +19 -0
- package/node_modules/playwright/package.json +55 -0
- package/node_modules/playwright/test.d.ts +18 -0
- package/node_modules/playwright/test.js +24 -0
- package/node_modules/playwright/test.mjs +35 -0
- package/node_modules/playwright/types/test.d.ts +11074 -0
- package/node_modules/playwright/types/testReporter.d.ts +950 -0
- package/node_modules/playwright-core/LICENSE +202 -0
- package/node_modules/playwright-core/NOTICE +5 -0
- package/node_modules/playwright-core/README.md +3 -0
- package/node_modules/playwright-core/ThirdPartyNotices.txt +13 -0
- package/node_modules/playwright-core/bin/install_media_pack.ps1 +5 -0
- package/node_modules/playwright-core/bin/install_webkit_wsl.ps1 +33 -0
- package/node_modules/playwright-core/bin/reinstall_chrome_beta_linux.sh +42 -0
- package/node_modules/playwright-core/bin/reinstall_chrome_beta_mac.sh +13 -0
- package/node_modules/playwright-core/bin/reinstall_chrome_beta_win.ps1 +24 -0
- package/node_modules/playwright-core/bin/reinstall_chrome_stable_linux.sh +42 -0
- package/node_modules/playwright-core/bin/reinstall_chrome_stable_mac.sh +12 -0
- package/node_modules/playwright-core/bin/reinstall_chrome_stable_win.ps1 +24 -0
- package/node_modules/playwright-core/bin/reinstall_msedge_beta_linux.sh +48 -0
- package/node_modules/playwright-core/bin/reinstall_msedge_beta_mac.sh +11 -0
- package/node_modules/playwright-core/bin/reinstall_msedge_beta_win.ps1 +23 -0
- package/node_modules/playwright-core/bin/reinstall_msedge_dev_linux.sh +48 -0
- package/node_modules/playwright-core/bin/reinstall_msedge_dev_mac.sh +11 -0
- package/node_modules/playwright-core/bin/reinstall_msedge_dev_win.ps1 +23 -0
- package/node_modules/playwright-core/bin/reinstall_msedge_stable_linux.sh +48 -0
- package/node_modules/playwright-core/bin/reinstall_msedge_stable_mac.sh +11 -0
- package/node_modules/playwright-core/bin/reinstall_msedge_stable_win.ps1 +24 -0
- package/node_modules/playwright-core/browsers.json +52 -0
- package/node_modules/playwright-core/cli.js +21 -0
- package/node_modules/playwright-core/index.d.ts +17 -0
- package/node_modules/playwright-core/index.js +17 -0
- package/node_modules/playwright-core/index.mjs +28 -0
- package/node_modules/playwright-core/lib/bootstrap.js +88 -0
- package/node_modules/playwright-core/lib/coreBundle.js +75813 -0
- package/node_modules/playwright-core/lib/entry/cliDaemon.js +5 -0
- package/node_modules/playwright-core/lib/entry/dashboardApp.js +3 -0
- package/node_modules/playwright-core/lib/entry/mcp.js +10 -0
- package/node_modules/playwright-core/lib/entry/oopBrowserDownload.js +3 -0
- package/node_modules/playwright-core/lib/package.js +50 -0
- package/node_modules/playwright-core/lib/server/chromium/appIcon.png +0 -0
- package/node_modules/playwright-core/lib/server/electron/loader.js +116 -0
- package/node_modules/playwright-core/lib/serverRegistry.js +1939 -0
- package/node_modules/playwright-core/lib/serverRegistry.js.LICENSE +65 -0
- package/node_modules/playwright-core/lib/tools/cli-client/channelSessions.js +141 -0
- package/node_modules/playwright-core/lib/tools/cli-client/cli.js +6 -0
- package/node_modules/playwright-core/lib/tools/cli-client/help.json +721 -0
- package/node_modules/playwright-core/lib/tools/cli-client/minimist.js +128 -0
- package/node_modules/playwright-core/lib/tools/cli-client/output.js +351 -0
- package/node_modules/playwright-core/lib/tools/cli-client/program.js +418 -0
- package/node_modules/playwright-core/lib/tools/cli-client/registry.js +180 -0
- package/node_modules/playwright-core/lib/tools/cli-client/session.js +259 -0
- package/node_modules/playwright-core/lib/tools/dashboard/appIcon.png +0 -0
- package/node_modules/playwright-core/lib/tools/skills/playwright-cli/SKILL.md +425 -0
- package/node_modules/playwright-core/lib/tools/skills/playwright-cli/references/element-attributes.md +23 -0
- package/node_modules/playwright-core/lib/tools/skills/playwright-cli/references/playwright-tests.md +39 -0
- package/node_modules/playwright-core/lib/tools/skills/playwright-cli/references/request-mocking.md +87 -0
- package/node_modules/playwright-core/lib/tools/skills/playwright-cli/references/running-code.md +241 -0
- package/node_modules/playwright-core/lib/tools/skills/playwright-cli/references/session-management.md +225 -0
- package/node_modules/playwright-core/lib/tools/skills/playwright-cli/references/storage-state.md +275 -0
- package/node_modules/playwright-core/lib/tools/skills/playwright-cli/references/test-generation.md +433 -0
- package/node_modules/playwright-core/lib/tools/skills/playwright-cli/references/tracing.md +139 -0
- package/node_modules/playwright-core/lib/tools/skills/playwright-cli/references/video-recording.md +143 -0
- package/node_modules/playwright-core/lib/tools/skills/playwright-component-testing/SKILL.md +136 -0
- package/node_modules/playwright-core/lib/tools/skills/playwright-component-testing/references/gallery-spec.md +144 -0
- package/node_modules/playwright-core/lib/tools/skills/playwright-component-testing/references/migration.md +89 -0
- package/node_modules/playwright-core/lib/tools/skills/playwright-component-testing/references/react.md +69 -0
- package/node_modules/playwright-core/lib/tools/skills/playwright-component-testing/references/typing.md +173 -0
- package/node_modules/playwright-core/lib/tools/skills/playwright-component-testing/references/vue.md +77 -0
- package/node_modules/playwright-core/lib/tools/skills/playwright-trace/SKILL.md +174 -0
- package/node_modules/playwright-core/lib/tools/utils/extension.js +112 -0
- package/node_modules/playwright-core/lib/tools/utils/socketConnection.js +108 -0
- package/node_modules/playwright-core/lib/utilsBundle.js +85357 -0
- package/node_modules/playwright-core/lib/utilsBundle.js.LICENSE +1890 -0
- package/node_modules/playwright-core/lib/vite/dashboard/assets/codeMirrorModule--QdMvsKi.css +1 -0
- package/node_modules/playwright-core/lib/vite/dashboard/assets/codeMirrorModule-CQ8RZGBm.js +32 -0
- package/node_modules/playwright-core/lib/vite/dashboard/assets/codicon-DCmgc-ay.ttf +0 -0
- package/node_modules/playwright-core/lib/vite/dashboard/assets/firefox-1bWoP6pv.svg +1 -0
- package/node_modules/playwright-core/lib/vite/dashboard/assets/firefox-beta-k3eOH_eK.svg +1 -0
- package/node_modules/playwright-core/lib/vite/dashboard/assets/firefox-nightly-Cp5nfeDT.svg +1 -0
- package/node_modules/playwright-core/lib/vite/dashboard/assets/index-AwgwcYie.css +1 -0
- package/node_modules/playwright-core/lib/vite/dashboard/assets/index-hRvO4yU2.js +12 -0
- package/node_modules/playwright-core/lib/vite/dashboard/assets/safari-na3_-uQk.svg +1 -0
- package/node_modules/playwright-core/lib/vite/dashboard/index.html +29 -0
- package/node_modules/playwright-core/lib/vite/dashboard/playwright-logo.svg +24 -0
- package/node_modules/playwright-core/lib/vite/htmlReport/index.html +16 -0
- package/node_modules/playwright-core/lib/vite/htmlReport/report.css +2 -0
- package/node_modules/playwright-core/lib/vite/htmlReport/report.js +32 -0
- package/node_modules/playwright-core/lib/vite/recorder/assets/codeMirrorModule--QdMvsKi.css +1 -0
- package/node_modules/playwright-core/lib/vite/recorder/assets/codeMirrorModule-CVQWJZAA.js +32 -0
- package/node_modules/playwright-core/lib/vite/recorder/assets/codicon-DCmgc-ay.ttf +0 -0
- package/node_modules/playwright-core/lib/vite/recorder/assets/index-BZpYJZ-P.css +1 -0
- package/node_modules/playwright-core/lib/vite/recorder/assets/index-hrlLqLtq.js +129 -0
- package/node_modules/playwright-core/lib/vite/recorder/index.html +29 -0
- package/node_modules/playwright-core/lib/vite/recorder/playwright-logo.svg +9 -0
- package/node_modules/playwright-core/lib/vite/traceViewer/assets/codeMirrorModule-BbkfBe3n.js +32 -0
- package/node_modules/playwright-core/lib/vite/traceViewer/assets/defaultSettingsView-Ds6CBOo0.js +182 -0
- package/node_modules/playwright-core/lib/vite/traceViewer/assets/urlMatch-L3liM589.js +1 -0
- package/node_modules/playwright-core/lib/vite/traceViewer/assets/xtermModule-DywYcAf8.js +7 -0
- package/node_modules/playwright-core/lib/vite/traceViewer/codeMirrorModule.-QdMvsKi.css +1 -0
- package/node_modules/playwright-core/lib/vite/traceViewer/codicon.DCmgc-ay.ttf +0 -0
- package/node_modules/playwright-core/lib/vite/traceViewer/defaultSettingsView.Bqk9acqE.css +1 -0
- package/node_modules/playwright-core/lib/vite/traceViewer/index.B4cLoZK3.js +1 -0
- package/node_modules/playwright-core/lib/vite/traceViewer/index.B_TqY17P.css +1 -0
- package/node_modules/playwright-core/lib/vite/traceViewer/index.html +44 -0
- package/node_modules/playwright-core/lib/vite/traceViewer/manifest.webmanifest +16 -0
- package/node_modules/playwright-core/lib/vite/traceViewer/playwright-logo.svg +9 -0
- package/node_modules/playwright-core/lib/vite/traceViewer/snapshot.B_Jk1wbt.js +1 -0
- package/node_modules/playwright-core/lib/vite/traceViewer/snapshot.html +10 -0
- package/node_modules/playwright-core/lib/vite/traceViewer/sw.bundle.js +4 -0
- package/node_modules/playwright-core/lib/vite/traceViewer/uiMode.CU5KtEkS.js +5 -0
- package/node_modules/playwright-core/lib/vite/traceViewer/uiMode.CyMfwkXJ.css +1 -0
- package/node_modules/playwright-core/lib/vite/traceViewer/uiMode.html +18 -0
- package/node_modules/playwright-core/lib/vite/traceViewer/xtermModule.kHJ-D0s7.css +1 -0
- package/node_modules/playwright-core/lib/webp_codec.LICENSE +173 -0
- package/node_modules/playwright-core/lib/webp_codec.wasm +0 -0
- package/node_modules/playwright-core/lib/xdg-open +1267 -0
- package/node_modules/playwright-core/package.json +34 -0
- package/node_modules/playwright-core/types/protocol.d.ts +24645 -0
- package/node_modules/playwright-core/types/structs.d.ts +52 -0
- package/node_modules/playwright-core/types/types.d.ts +26632 -0
- package/package.json +12 -2
- package/plugins/devflow-core/extensions/devflow-core.ts +431 -7
- package/plugins/devflow-core/runtime/hooks/hook_common.py +269 -0
- package/plugins/devflow-core/runtime/hooks/index_gate.py +29 -16
- package/plugins/devflow-core/skills/workflow-orchestrator/scripts/acceptance_test_runner.py +407 -71
- package/plugins/devflow-core/skills/workflow-orchestrator/scripts/advance_state.rb +51 -0
- package/plugins/devflow-core/skills/workflow-orchestrator/scripts/approval_pins.py +433 -0
- package/plugins/devflow-core/skills/workflow-orchestrator/scripts/approval_store.py +137 -0
- package/plugins/devflow-core/skills/workflow-orchestrator/scripts/dsl_coverage.py +87 -0
- package/plugins/devflow-core/skills/workflow-orchestrator/scripts/dsl_v1.py +318 -0
- package/plugins/devflow-core/skills/workflow-orchestrator/scripts/environment_profile.py +134 -0
- package/plugins/devflow-core/skills/workflow-orchestrator/scripts/execution_contract.rb +1 -0
- package/plugins/devflow-core/skills/workflow-orchestrator/scripts/execution_surface.py +138 -0
- package/plugins/devflow-core/skills/workflow-orchestrator/scripts/playwright_adapter.py +515 -0
- package/plugins/devflow-core/skills/workflow-orchestrator/scripts/tc_hash.py +132 -0
- package/plugins/devflow-core/skills/workflow-orchestrator/scripts/test_command_policy.py +102 -8
- package/plugins/devflow-core/skills/workflow-orchestrator/scripts/validate_state.rb +13 -0
- package/plugins/devflow-core/skills/workflow-orchestrator/templates/feature-development.yaml +13 -34
- package/plugins/devflow-core/skills/workflow-pipeline/SKILL.md +3 -1
- package/plugins/devflow-core/skills/workflow-pipeline/references/pipeline-flow.md +68 -140
- package/plugins/devflow-core/skills/workflow-project-context/scripts/validate_initialization_database.py +1 -1
- package/plugins/devflow-implementation/skills/workflow-execute-phase/references/execute-phase.md +3 -1
- package/plugins/devflow-intel/skills/workflow-code-index/scripts/code_index.py +23 -4
- package/plugins/devflow-quality/runtime/playwright/devflow-test-dsl-v1.schema.json +872 -0
- package/plugins/devflow-quality/runtime/playwright/playwright.config.mjs +46 -0
- package/plugins/devflow-quality/runtime/playwright/reporter.mjs +311 -0
- package/plugins/devflow-quality/runtime/playwright/runner.spec.mjs +1308 -0
- package/plugins/devflow-quality/runtime/playwright/runtime-manifest.json +11 -0
- package/plugins/devflow-quality/skills/workflow-acceptance-test-design/SKILL.md +3 -0
- package/plugins/devflow-quality/skills/workflow-generate-mainflow-cases/SKILL.md +71 -58
- package/plugins/devflow-quality/skills/workflow-generate-mainflow-cases/references/api-mainflow-rules.md +20 -15
- package/plugins/devflow-quality/skills/workflow-generate-mainflow-cases/references/codebase-discovery.md +3 -1
- package/plugins/devflow-quality/skills/workflow-generate-mainflow-cases/references/intermediate-format.md +115 -192
- package/plugins/devflow-quality/skills/workflow-generate-mainflow-cases/references/mainflow-selection.md +27 -23
- package/plugins/devflow-quality/skills/workflow-generate-mainflow-cases/references/project-dsl-contract.md +78 -153
- package/plugins/devflow-quality/skills/workflow-generate-mainflow-cases/references/ui-mainflow-rules.md +36 -83
- package/plugins/devflow-quality/skills/workflow-generate-mainflow-cases/scripts/compile_dsl.py +497 -556
- package/plugins/devflow-quality/skills/workflow-generate-mainflow-cases/scripts/export_legacy_testagent.py +647 -0
- package/plugins/devflow-quality/skills/workflow-generate-mainflow-cases/scripts/normalize_case_payloads.py +10 -3
- package/plugins/devflow-quality/skills/workflow-generate-mainflow-cases/scripts/validate_case_payloads.py +165 -275
- package/plugins/devflow-quality/skills/workflow-human-acceptance/SKILL.md +61 -0
- package/plugins/devflow-quality/skills/workflow-release-closeout/SKILL.md +4 -4
- package/plugins/devflow-quality/skills/workflow-run-acceptance-tests/SKILL.md +15 -9
- package/src/approval-store.ts +340 -0
- package/src/constants.ts +1 -1
- package/src/doctor.ts +259 -2
- package/src/initialization-store.ts +48 -1
- package/src/paths.ts +12 -1
- package/src/project-context.ts +24 -1
- package/templates/omp/hook-config.json +6 -0
|
@@ -7,14 +7,22 @@ import hashlib
|
|
|
7
7
|
import json
|
|
8
8
|
import os
|
|
9
9
|
import re
|
|
10
|
+
import shutil
|
|
10
11
|
import subprocess
|
|
11
|
-
from datetime import datetime, timezone
|
|
12
|
+
from datetime import date, datetime, timedelta, timezone
|
|
12
13
|
from pathlib import Path
|
|
13
14
|
from typing import Any
|
|
14
15
|
from urllib.error import HTTPError, URLError
|
|
15
16
|
from urllib.parse import urlencode, urljoin
|
|
16
17
|
from urllib.request import Request, urlopen
|
|
17
18
|
|
|
19
|
+
import playwright_adapter
|
|
20
|
+
from approval_store import REQUIRES_DATABASE, valid_approval
|
|
21
|
+
from dsl_coverage import check as dsl_coverage_check
|
|
22
|
+
from dsl_v1 import validate as validate_dsl
|
|
23
|
+
from environment_profile import load as load_environment_profile
|
|
24
|
+
from execution_surface import ADAPTERS_BY_SURFACE, consistency_errors, derive as derive_surface, http_execution_error
|
|
25
|
+
from tc_hash import tc_sha256
|
|
18
26
|
from test_command_policy import validate_test_command
|
|
19
27
|
|
|
20
28
|
|
|
@@ -22,6 +30,11 @@ FEATURE_RE = re.compile(r"^[^/\\\x00-\x1f\x7f]+$")
|
|
|
22
30
|
AUTOMATION_MODES = {"automated", "manual", "blocked"}
|
|
23
31
|
EVIDENCE_STRATEGIES = {"replay", "causal", "observation"}
|
|
24
32
|
MAX_EVIDENCE_FILE_BYTES = 25 * 1024 * 1024
|
|
33
|
+
# 方案 §11: automation_mode may only be downgraded by a binding, never upgraded.
|
|
34
|
+
ALLOWED_BINDING_MODES = {"automated": ("automated", "manual", "blocked"), "manual": ("manual", "blocked"), "blocked": ("blocked",)}
|
|
35
|
+
RESULT_ORIGINS = {"command": "project_test_command", "http": "core_http_adapter", playwright_adapter.ADAPTER_KIND: playwright_adapter.RESULT_ORIGIN}
|
|
36
|
+
DEFAULT_RUN_RETENTION_DAYS = 30
|
|
37
|
+
DEFAULT_UNREFERENCED_RUN_TTL_DAYS = 7
|
|
25
38
|
|
|
26
39
|
|
|
27
40
|
def utc_now() -> str:
|
|
@@ -56,6 +69,49 @@ def write_json(path: Path, value: dict[str, Any]) -> None:
|
|
|
56
69
|
temporary.replace(path)
|
|
57
70
|
|
|
58
71
|
|
|
72
|
+
def human_results_path(root: Path, feature: str) -> Path:
|
|
73
|
+
"""Agent-maintained draft/evidence list only; never a result source (方案 §11)."""
|
|
74
|
+
return root / ".devflow/planning" / "test-runs" / feature / "human-acceptance-results.json"
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def manual_case_ids(root: Path, feature: str, suite: dict[str, Any], bindings: dict[str, Any] | None) -> list[str]:
|
|
78
|
+
"""Manual TCs are determined by the binding mode; before bindings exist, by the frozen mode."""
|
|
79
|
+
cases = frozen_cases(suite, [])
|
|
80
|
+
if bindings is None:
|
|
81
|
+
_, binding_path, _ = paths(root, feature)
|
|
82
|
+
bindings = read_json(binding_path, [], "acceptance-test bindings") if binding_path.is_file() else None
|
|
83
|
+
if isinstance(bindings, dict) and isinstance(bindings.get("cases"), list):
|
|
84
|
+
return sorted({entry["tc_id"] for entry in bindings["cases"] if isinstance(entry, dict) and entry.get("mode") == "manual" and entry.get("tc_id") in cases})
|
|
85
|
+
return sorted(tc_id for tc_id, data in cases.items() if data.get("automation_mode") == "manual")
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def validate_human_results(root: Path, feature: str, suite: dict[str, Any], bindings: dict[str, Any] | None = None) -> tuple[dict[str, Any], dict[str, dict[str, Any]]]:
|
|
89
|
+
"""Manual results come only from valid Core human_result approvals (UI-confirmed, pins re-derived)."""
|
|
90
|
+
errors: list[str] = []
|
|
91
|
+
manual = manual_case_ids(root, feature, suite, bindings)
|
|
92
|
+
by_id: dict[str, dict[str, Any]] = {}
|
|
93
|
+
for tc_id in manual:
|
|
94
|
+
found = valid_approval(root, feature, "human_result", tc_id)
|
|
95
|
+
if not found["ok"]:
|
|
96
|
+
code = REQUIRES_DATABASE if found["reason"] == REQUIRES_DATABASE else "human_result_unapproved"
|
|
97
|
+
errors.append(f"{code}: manual TC {tc_id} has no valid human_result approval ({found['message']})")
|
|
98
|
+
continue
|
|
99
|
+
approval = found["approval"]
|
|
100
|
+
decision = approval["decision"]
|
|
101
|
+
status = decision.get("status")
|
|
102
|
+
actual = decision.get("actual_result")
|
|
103
|
+
evidence = approval["pins"].get("evidence") if isinstance(approval["pins"].get("evidence"), list) else []
|
|
104
|
+
if status not in {"passed", "failed", "blocked"} or not isinstance(actual, str) or not actual.strip():
|
|
105
|
+
errors.append(f"approval_invalid: human_result approval for {tc_id} must record status and actual_result")
|
|
106
|
+
continue
|
|
107
|
+
if status == "passed" and not evidence:
|
|
108
|
+
errors.append(f"approval_invalid: passed human_result for {tc_id} requires at least one pinned evidence file")
|
|
109
|
+
continue
|
|
110
|
+
by_id[tc_id] = {"status": status, "actual_result": actual, "approval_id": approval["id"], "approved_at": approval["approved_at"],
|
|
111
|
+
"evidence_artifacts": [{"path": item.get("path"), "sha256": item.get("sha256")} for item in evidence if isinstance(item, dict)]}
|
|
112
|
+
return {"ok": not errors, "feature": feature, "tc_ids": manual, "errors": errors}, by_id
|
|
113
|
+
|
|
114
|
+
|
|
59
115
|
def valid_argv(value: Any) -> bool:
|
|
60
116
|
return isinstance(value, list) and bool(value) and all(isinstance(token, str) and token and "\x00" not in token for token in value)
|
|
61
117
|
|
|
@@ -90,30 +146,6 @@ def pointer_value(document: Any, pointer: str) -> Any:
|
|
|
90
146
|
return current
|
|
91
147
|
|
|
92
148
|
|
|
93
|
-
def http_execution_error(source: dict[str, Any]) -> str | None:
|
|
94
|
-
execution = source.get("execution")
|
|
95
|
-
if not isinstance(execution, dict) or execution.get("protocol") != "http":
|
|
96
|
-
return "frozen TC has no HTTP execution definition"
|
|
97
|
-
request = execution.get("request")
|
|
98
|
-
expect = execution.get("expect")
|
|
99
|
-
if not isinstance(request, dict) or not isinstance(expect, dict):
|
|
100
|
-
return "frozen TC HTTP execution requires request and expect"
|
|
101
|
-
if not isinstance(request.get("method"), str) or not re.fullmatch(r"[A-Z]+", request["method"]):
|
|
102
|
-
return "frozen TC HTTP request method is invalid"
|
|
103
|
-
if not isinstance(request.get("path"), str) or not request["path"].startswith("/"):
|
|
104
|
-
return "frozen TC HTTP request path is invalid"
|
|
105
|
-
if not isinstance(expect.get("http_status"), int) or not 100 <= expect["http_status"] <= 599:
|
|
106
|
-
return "frozen TC HTTP expectation http_status is invalid"
|
|
107
|
-
for key in ("path_params", "query", "headers", "body"):
|
|
108
|
-
if key in request and not isinstance(request[key], dict):
|
|
109
|
-
return f"frozen TC HTTP request {key} must be an object"
|
|
110
|
-
if "error_code" in expect and (not isinstance(expect["error_code"], str) or not expect["error_code"].strip()):
|
|
111
|
-
return "frozen TC HTTP expectation error_code is invalid"
|
|
112
|
-
if "error_code_pointer" in expect and (not isinstance(expect["error_code_pointer"], str) or not expect["error_code_pointer"].startswith("/")):
|
|
113
|
-
return "frozen TC HTTP expectation error_code_pointer is invalid"
|
|
114
|
-
return None
|
|
115
|
-
|
|
116
|
-
|
|
117
149
|
def frozen_cases(suite: dict[str, Any], errors: list[str]) -> dict[str, dict[str, Any]]:
|
|
118
150
|
cases: dict[str, dict[str, Any]] = {}
|
|
119
151
|
for node in suite.get("nodes", []):
|
|
@@ -144,27 +176,33 @@ def file_fingerprint(path: Path) -> str | None:
|
|
|
144
176
|
return digest(path.read_bytes()) if path.is_file() else None
|
|
145
177
|
|
|
146
178
|
|
|
147
|
-
def
|
|
148
|
-
"""
|
|
179
|
+
def evidence_declaration_errors(root: Path, value: Any) -> list[str]:
|
|
180
|
+
"""Before a run only the declaration and path boundary are checked; files may be produced by the run."""
|
|
149
181
|
if not isinstance(value, list) or not value:
|
|
150
|
-
return [
|
|
182
|
+
return ["causal verification requires non-empty evidence_files"]
|
|
151
183
|
root_real = root.resolve()
|
|
152
|
-
artifacts: list[dict[str, Any]] = []
|
|
153
184
|
errors: list[str] = []
|
|
154
185
|
for raw in value:
|
|
155
186
|
if not isinstance(raw, str) or not raw.strip() or "\x00" in raw:
|
|
156
187
|
errors.append("evidence_files entries must be non-empty relative paths")
|
|
157
188
|
continue
|
|
158
189
|
candidate = Path(raw)
|
|
159
|
-
if candidate.is_absolute() or ".." in candidate.parts:
|
|
190
|
+
if candidate.is_absolute() or ".." in candidate.parts or not (root_real / candidate).resolve().is_relative_to(root_real):
|
|
160
191
|
errors.append(f"evidence file escapes project root: {raw}")
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
192
|
+
return errors
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def evidence_artifacts(root: Path, value: Any) -> tuple[list[dict[str, Any]], list[str]]:
|
|
196
|
+
"""After a run: hash causal evidence from bounded project-relative files only."""
|
|
197
|
+
declaration_errors = evidence_declaration_errors(root, value)
|
|
198
|
+
if declaration_errors:
|
|
199
|
+
return [], declaration_errors
|
|
200
|
+
root_real = root.resolve()
|
|
201
|
+
artifacts: list[dict[str, Any]] = []
|
|
202
|
+
errors: list[str] = []
|
|
203
|
+
for raw in value:
|
|
204
|
+
resolved = (root_real / Path(raw)).resolve()
|
|
205
|
+
relative = resolved.relative_to(root_real)
|
|
168
206
|
if not resolved.is_file():
|
|
169
207
|
errors.append(f"causal evidence file is missing: {raw}")
|
|
170
208
|
continue
|
|
@@ -237,6 +275,37 @@ def run_http_case(source: dict[str, Any], adapter: dict[str, Any]) -> tuple[bool
|
|
|
237
275
|
return True, 0, f"{setup_output}\nHTTP {http_status} matched expectation"
|
|
238
276
|
|
|
239
277
|
|
|
278
|
+
def playwright_binding_errors(root: Path, feature: str, tc_id: str, source: dict[str, Any], adapter: dict[str, Any]) -> list[str]:
|
|
279
|
+
"""Binding shape, DSL schema/semantics, TC identity, structural coverage and profile validity."""
|
|
280
|
+
errors = playwright_adapter.adapter_errors(root, feature, adapter)
|
|
281
|
+
if errors:
|
|
282
|
+
return errors
|
|
283
|
+
path, _ = playwright_adapter.dsl_path(root, feature, adapter["dsl_file"])
|
|
284
|
+
assert path is not None
|
|
285
|
+
_raw, dsl, dsl_error = playwright_adapter.read_dsl(path)
|
|
286
|
+
if dsl_error:
|
|
287
|
+
return [dsl_error]
|
|
288
|
+
dsl_errors = validate_dsl(dsl)
|
|
289
|
+
if dsl_errors:
|
|
290
|
+
return [f"DSL is invalid: {error}" for error in dsl_errors]
|
|
291
|
+
if dsl["tc_id"] != tc_id:
|
|
292
|
+
errors.append(f"DSL tc_id {dsl['tc_id']} does not match binding {tc_id}")
|
|
293
|
+
if dsl["tc_sha256"] != tc_sha256(source):
|
|
294
|
+
errors.append("DSL tc_sha256 does not match the current frozen TC (JCS SHA-256)")
|
|
295
|
+
errors.extend(f"DSL coverage: {error}" for error in dsl_coverage_check(source, dsl))
|
|
296
|
+
_profile, _sha, profile_errors = load_environment_profile(root, adapter["environment_profile"])
|
|
297
|
+
errors.extend(f"environment profile: {error}" for error in profile_errors)
|
|
298
|
+
return errors
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
def mode_override_errors(root: Path, feature: str, tc_id: str) -> list[str]:
|
|
302
|
+
found = valid_approval(root, feature, "mode_override", tc_id)
|
|
303
|
+
if found["ok"]:
|
|
304
|
+
return []
|
|
305
|
+
code = REQUIRES_DATABASE if found["reason"] == REQUIRES_DATABASE else "approval_invalid"
|
|
306
|
+
return [f"{code}: automated -> manual downgrade of {tc_id} requires a valid UI-confirmed mode_override approval ({found['message']})"]
|
|
307
|
+
|
|
308
|
+
|
|
240
309
|
def validate(root: Path, feature: str) -> tuple[dict[str, Any], dict[str, Any] | None, dict[str, Any] | None]:
|
|
241
310
|
errors: list[str] = []
|
|
242
311
|
suite_path, binding_path, _ = paths(root, feature)
|
|
@@ -282,6 +351,8 @@ def validate(root: Path, feature: str) -> tuple[dict[str, Any], dict[str, Any] |
|
|
|
282
351
|
if source is None:
|
|
283
352
|
errors.append(f"unknown TC reference: {tc_id}")
|
|
284
353
|
continue
|
|
354
|
+
surface, _legacy = derive_surface(source)
|
|
355
|
+
errors.extend(f"execution_surface_mismatch: {label} ({tc_id}) {error}" for error in consistency_errors(source))
|
|
285
356
|
strategy = source.get("evidence_strategy")
|
|
286
357
|
if strategy is not None and strategy not in EVIDENCE_STRATEGIES:
|
|
287
358
|
errors.append(f"{label}.source evidence_strategy is invalid")
|
|
@@ -291,20 +362,30 @@ def validate(root: Path, feature: str) -> tuple[dict[str, Any], dict[str, Any] |
|
|
|
291
362
|
if mode not in AUTOMATION_MODES:
|
|
292
363
|
errors.append(f"{label}.mode is invalid")
|
|
293
364
|
continue
|
|
294
|
-
|
|
295
|
-
|
|
365
|
+
frozen_mode = source.get("automation_mode")
|
|
366
|
+
if mode not in ALLOWED_BINDING_MODES.get(frozen_mode, ()):
|
|
367
|
+
errors.append(f"{label}.mode {mode} is not allowed for frozen automation_mode {frozen_mode} (bindings may only downgrade)")
|
|
368
|
+
continue
|
|
296
369
|
if mode == "automated":
|
|
297
370
|
adapter = entry.get("adapter")
|
|
298
371
|
if not isinstance(adapter, dict):
|
|
299
372
|
errors.append(f"{label}.adapter is required")
|
|
300
|
-
|
|
373
|
+
continue
|
|
374
|
+
kind = adapter.get("kind")
|
|
375
|
+
if kind not in RESULT_ORIGINS:
|
|
376
|
+
errors.append(f"{label}.adapter kind must be command, http, or devflow_playwright")
|
|
377
|
+
continue
|
|
378
|
+
if surface is not None and kind not in ADAPTERS_BY_SURFACE[surface]:
|
|
379
|
+
errors.append(f"execution_surface_mismatch: {label} ({tc_id}) execution_surface={surface} cannot be bound to adapter kind {kind}")
|
|
380
|
+
continue
|
|
381
|
+
if kind == "command":
|
|
301
382
|
if not valid_argv(adapter.get("argv")):
|
|
302
383
|
errors.append(f"{label}.adapter command requires non-empty argv")
|
|
303
384
|
continue
|
|
304
385
|
command_error = validate_test_command(root, adapter["argv"])
|
|
305
386
|
if command_error:
|
|
306
387
|
errors.append(f"{label}.adapter: {command_error}")
|
|
307
|
-
elif
|
|
388
|
+
elif kind == "http":
|
|
308
389
|
if not valid_http_adapter(adapter):
|
|
309
390
|
errors.append(f"{label}.adapter HTTP requires base_url_env, valid optional headers_env, and optional setup.argv")
|
|
310
391
|
else:
|
|
@@ -316,27 +397,45 @@ def validate(root: Path, feature: str) -> tuple[dict[str, Any], dict[str, Any] |
|
|
|
316
397
|
if setup_error:
|
|
317
398
|
errors.append(f"{label}.adapter.setup: {setup_error}")
|
|
318
399
|
else:
|
|
319
|
-
errors.
|
|
400
|
+
errors.extend(f"{label}.adapter: {error}" for error in playwright_binding_errors(root, feature, tc_id, source, adapter))
|
|
320
401
|
if strategy == "causal":
|
|
321
|
-
|
|
322
|
-
errors.extend(f"{label}.{error}" for error in artifact_errors)
|
|
402
|
+
errors.extend(f"{label}.{error}" for error in evidence_declaration_errors(root, entry.get("evidence_files")))
|
|
323
403
|
elif mode == "manual":
|
|
324
404
|
if not isinstance(entry.get("owner"), str) or not entry["owner"].strip() or not isinstance(entry.get("reason"), str) or not entry["reason"].strip():
|
|
325
405
|
errors.append(f"{label}.manual requires owner and reason")
|
|
406
|
+
if frozen_mode == "automated":
|
|
407
|
+
errors.extend(f"{error} [{label}]" for error in mode_override_errors(root, feature, tc_id))
|
|
326
408
|
elif mode == "blocked":
|
|
327
409
|
if not isinstance(entry.get("blocked_by"), str) or not entry["blocked_by"].strip() or not isinstance(entry.get("owner"), str) or not entry["owner"].strip():
|
|
328
410
|
errors.append(f"{label}.blocked requires blocked_by and owner")
|
|
411
|
+
if frozen_mode != "blocked" and (not isinstance(entry.get("reason"), str) or not entry["reason"].strip()):
|
|
412
|
+
errors.append(f"{label}.blocked downgrade requires reason")
|
|
329
413
|
for tc_id in sorted(set(cases) - seen):
|
|
330
414
|
errors.append(f"unbound frozen TC: {tc_id}")
|
|
415
|
+
human_validation, _ = validate_human_results(root, feature, suite, bindings)
|
|
416
|
+
errors.extend(human_validation["errors"])
|
|
331
417
|
return {"ok": not errors, "feature": feature, "tc_ids": sorted(cases), "errors": errors}, suite, bindings
|
|
332
418
|
|
|
333
419
|
|
|
420
|
+
def run_command_case(root: Path, argv: list[str]) -> tuple[int, str]:
|
|
421
|
+
try:
|
|
422
|
+
completed = subprocess.run(argv, cwd=root, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=180, check=False)
|
|
423
|
+
return completed.returncode, f"{completed.stdout}\n{completed.stderr}"
|
|
424
|
+
except subprocess.TimeoutExpired as error:
|
|
425
|
+
return 124, f"command timed out: {error}"
|
|
426
|
+
except OSError as error:
|
|
427
|
+
return 127, f"command could not start: {error}"
|
|
428
|
+
|
|
429
|
+
|
|
334
430
|
def run(root: Path, feature: str, write: bool) -> dict[str, Any]:
|
|
335
431
|
validation, suite, bindings = validate(root, feature)
|
|
336
432
|
if not validation["ok"]:
|
|
337
433
|
return validation
|
|
338
434
|
assert suite is not None and bindings is not None
|
|
339
435
|
cases = frozen_cases(suite, [])
|
|
436
|
+
human_validation, human_results = validate_human_results(root, feature, suite, bindings)
|
|
437
|
+
if not human_validation["ok"]:
|
|
438
|
+
return human_validation
|
|
340
439
|
_, _, result_path = paths(root, feature)
|
|
341
440
|
previous = read_json(result_path, [], "previous acceptance-test results")
|
|
342
441
|
task_bindings_path = root / ".devflow/planning" / "features" / feature / "test-bindings.json"
|
|
@@ -345,43 +444,54 @@ def run(root: Path, feature: str, write: bool) -> dict[str, Any]:
|
|
|
345
444
|
for entry in bindings["cases"]:
|
|
346
445
|
tc_id = entry["tc_id"]
|
|
347
446
|
mode = entry["mode"]
|
|
348
|
-
|
|
447
|
+
source = cases[tc_id]
|
|
448
|
+
strategy = source.get("evidence_strategy")
|
|
449
|
+
surface, legacy = derive_surface(source)
|
|
450
|
+
common = {"tc_id": tc_id, "mode": mode, "execution_surface": surface, "legacy_unclassified": legacy, "tc_sha256": tc_sha256(source)}
|
|
451
|
+
frozen_mode = source.get("automation_mode")
|
|
452
|
+
if mode != frozen_mode:
|
|
453
|
+
common["downgrade"] = {"from": frozen_mode, "to": mode}
|
|
349
454
|
if mode == "automated":
|
|
350
455
|
started_at = utc_now()
|
|
351
456
|
adapter = entry["adapter"]
|
|
352
|
-
|
|
457
|
+
kind = adapter["kind"]
|
|
458
|
+
if kind == "command":
|
|
353
459
|
argv = adapter["argv"]
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
exit_code = completed.returncode
|
|
357
|
-
output = f"{completed.stdout}\n{completed.stderr}"
|
|
358
|
-
except subprocess.TimeoutExpired as error:
|
|
359
|
-
exit_code = 124
|
|
360
|
-
output = f"command timed out: {error}"
|
|
361
|
-
except OSError as error:
|
|
362
|
-
exit_code = 127
|
|
363
|
-
output = f"command could not start: {error}"
|
|
364
|
-
result = {"tc_id": tc_id, "status": "passed" if exit_code == 0 else "failed", "mode": mode, "adapter_kind": "command", "argv": argv,
|
|
460
|
+
exit_code, output = run_command_case(root, argv)
|
|
461
|
+
result = {**common, "status": "passed" if exit_code == 0 else "failed", "adapter_kind": "command", "origin": RESULT_ORIGINS[kind], "argv": argv,
|
|
365
462
|
"command_hash": digest(json.dumps(argv, ensure_ascii=False, separators=(",", ":"))), "started_at": started_at,
|
|
366
463
|
"finished_at": utc_now(), "exit_code": exit_code, "output_sha256": digest(output)}
|
|
367
|
-
|
|
368
|
-
passed, exit_code, output = run_http_case(
|
|
369
|
-
result = {
|
|
464
|
+
elif kind == "http":
|
|
465
|
+
passed, exit_code, output = run_http_case(source, adapter)
|
|
466
|
+
result = {**common, "status": "passed" if passed else "failed", "adapter_kind": "http", "origin": RESULT_ORIGINS[kind],
|
|
370
467
|
"adapter_fingerprint": digest(json.dumps(adapter, ensure_ascii=False, separators=(",", ":"))), "started_at": started_at,
|
|
371
468
|
"finished_at": utc_now(), "exit_code": exit_code, "output_sha256": digest(output)}
|
|
469
|
+
else:
|
|
470
|
+
outcome = playwright_adapter.execute(root, feature, tc_id, source, suite, adapter)
|
|
471
|
+
result = {**common, "adapter_kind": kind, "origin": RESULT_ORIGINS[kind], "started_at": started_at, **outcome}
|
|
472
|
+
result.setdefault("finished_at", utc_now())
|
|
473
|
+
result.setdefault("dsl_approval", "none")
|
|
372
474
|
if strategy == "causal":
|
|
373
475
|
artifacts, artifact_errors = evidence_artifacts(root, entry.get("evidence_files"))
|
|
374
476
|
if artifact_errors:
|
|
375
|
-
result["status"]
|
|
477
|
+
if result["status"] == "passed":
|
|
478
|
+
result["status"] = "failed"
|
|
376
479
|
result["evidence_error"] = "; ".join(artifact_errors)
|
|
377
480
|
else:
|
|
378
481
|
result["evidence_artifacts"] = artifacts
|
|
379
482
|
results.append(result)
|
|
380
483
|
elif mode == "manual":
|
|
381
|
-
|
|
382
|
-
|
|
484
|
+
human = human_results[tc_id]
|
|
485
|
+
if "downgrade" in common:
|
|
486
|
+
override = valid_approval(root, feature, "mode_override", tc_id)
|
|
487
|
+
common["downgrade"]["approval_id"] = override["approval"]["id"] if override["ok"] else None
|
|
488
|
+
results.append({**common, "status": human["status"], "origin": "human_result_approval", "approval_id": human["approval_id"],
|
|
489
|
+
"approved_at": human["approved_at"], "actual_result": human["actual_result"], "evidence_artifacts": human["evidence_artifacts"]})
|
|
383
490
|
else:
|
|
384
|
-
|
|
491
|
+
blocked_result = {**common, "status": "blocked", "owner": entry["owner"], "blocked_by": entry["blocked_by"]}
|
|
492
|
+
if isinstance(entry.get("reason"), str):
|
|
493
|
+
blocked_result["reason"] = entry["reason"]
|
|
494
|
+
results.append(blocked_result)
|
|
385
495
|
all_passed = all(item["status"] == "passed" for item in results)
|
|
386
496
|
history = previous.get("history", []) if isinstance(previous, dict) and isinstance(previous.get("history"), list) else []
|
|
387
497
|
if isinstance(previous, dict) and isinstance(previous.get("results"), list):
|
|
@@ -390,7 +500,9 @@ def run(root: Path, feature: str, write: bool) -> dict[str, Any]:
|
|
|
390
500
|
"results": previous["results"]}]
|
|
391
501
|
payload = {"schema_version": 1, "origin": "core_acceptance_test_runner", "feature": feature, "executed_at": utc_now(),
|
|
392
502
|
"test_suite": bindings["test_suite"], "implementation_task_bindings": {"path": str(task_bindings_path.relative_to(root)), "fingerprint": file_fingerprint(task_bindings_path)},
|
|
393
|
-
"execution_binding_fingerprint": file_fingerprint(execution_bindings_path),
|
|
503
|
+
"execution_binding_fingerprint": file_fingerprint(execution_bindings_path),
|
|
504
|
+
"human_acceptance_fingerprint": file_fingerprint(human_results_path(root, feature)),
|
|
505
|
+
"results": results, "passed": all_passed, "history": history[-20:]}
|
|
394
506
|
if write:
|
|
395
507
|
write_json(result_path, payload)
|
|
396
508
|
meta_path = result_path.with_name("acceptance-results.meta.json")
|
|
@@ -403,6 +515,9 @@ def run(root: Path, feature: str, write: bool) -> dict[str, Any]:
|
|
|
403
515
|
"acceptance_tc_passed": passed,
|
|
404
516
|
"acceptance_tc_failed": failed,
|
|
405
517
|
"acceptance_tc_blocked": blocked,
|
|
518
|
+
"acceptance_tc_legacy_unclassified": sum(1 for item in results if item.get("legacy_unclassified")),
|
|
519
|
+
"acceptance_tc_downgraded": sum(1 for item in results if item.get("downgrade")),
|
|
520
|
+
"acceptance_tc_dsl_unapproved": sum(1 for item in results if item.get("dsl_approval") == "none"),
|
|
406
521
|
"acceptance_suite_graph_revision": payload["test_suite"].get("graph_revision"),
|
|
407
522
|
"acceptance_results_fingerprint": file_fingerprint(result_path),
|
|
408
523
|
"built_by": "core_acceptance_test_runner",
|
|
@@ -412,13 +527,144 @@ def run(root: Path, feature: str, write: bool) -> dict[str, Any]:
|
|
|
412
527
|
"errors": [] if all_passed else ["one or more frozen TC cases failed or are blocked"]}
|
|
413
528
|
|
|
414
529
|
|
|
415
|
-
def
|
|
416
|
-
|
|
530
|
+
def run_evidence_expired(root: Path, feature: str, today: date | None = None) -> bool:
|
|
531
|
+
"""True only once release-closeout completed and the post-closeout retention window has elapsed.
|
|
532
|
+
|
|
533
|
+
Decision: a missing .devflow/runs/<feature>/<execution_id> directory is an error while the feature is
|
|
534
|
+
open or within DEFAULT_RUN_RETENTION_DAYS after closeout (gc_runs never removes referenced runs
|
|
535
|
+
earlier, so a missing directory then means deletion/tampering). After that window gc-runs may have
|
|
536
|
+
removed it legitimately, so only the file-level evidence checks are skipped; every other recorded
|
|
537
|
+
field is still re-verified against current state.
|
|
538
|
+
"""
|
|
539
|
+
closed = closeout_completed_on(root, feature)
|
|
540
|
+
return closed is not None and (today or datetime.now(timezone.utc).date()) >= closed + timedelta(days=DEFAULT_RUN_RETENTION_DAYS)
|
|
541
|
+
|
|
542
|
+
|
|
543
|
+
def _recorded_str(entry: dict[str, Any], key: str) -> str | None:
|
|
544
|
+
value = entry.get(key)
|
|
545
|
+
return value if isinstance(value, str) and value else None
|
|
546
|
+
|
|
547
|
+
|
|
548
|
+
def playwright_entry_errors(root: Path, feature: str, tc_id: str, source: dict[str, Any], suite: dict[str, Any], adapter: dict[str, Any],
|
|
549
|
+
entry: dict[str, Any], run_dir_expired: bool) -> list[str]:
|
|
550
|
+
"""A devflow_playwright result must still be backed by the CURRENT execution_plan and environment_profile approvals."""
|
|
551
|
+
errors: list[str] = []
|
|
552
|
+
if entry.get("origin") != playwright_adapter.RESULT_ORIGIN:
|
|
553
|
+
errors.append(f"origin must be {playwright_adapter.RESULT_ORIGIN}")
|
|
554
|
+
runtime = entry.get("runtime")
|
|
555
|
+
if not isinstance(runtime, dict) or not all(isinstance(runtime.get(key), str) and runtime[key] for key in ("version", "manifest_sha256")):
|
|
556
|
+
errors.append("runtime info (version, manifest_sha256) is missing")
|
|
557
|
+
for key in ("execution_id", "run_dir"):
|
|
558
|
+
if _recorded_str(entry, key) is None:
|
|
559
|
+
errors.append(f"{key} is missing")
|
|
560
|
+
if not (isinstance(entry.get("result_sha256"), str) and playwright_adapter.SHA256_RE.fullmatch(entry["result_sha256"])):
|
|
561
|
+
errors.append("result_sha256 is missing or malformed")
|
|
562
|
+
approval = entry.get("dsl_approval")
|
|
563
|
+
if approval == "none" or not isinstance(approval, dict) or approval.get("status") != "approved" or not _recorded_str(approval, "approval_id"):
|
|
564
|
+
errors.append(f"dsl_approval is {approval!r}, not an approved execution_plan approval")
|
|
565
|
+
if playwright_adapter.adapter_errors(root, feature, adapter):
|
|
566
|
+
return errors # the binding itself is invalid; validate() already reported why
|
|
567
|
+
trust = playwright_adapter.resolve_trust(root, feature, tc_id, source, suite, adapter)
|
|
568
|
+
current = trust["base"]
|
|
569
|
+
if trust["failure"] is not None:
|
|
570
|
+
failure = trust["failure"]
|
|
571
|
+
return errors + [f"{failure.get('error_code')}: {failure.get('error')}"]
|
|
572
|
+
if entry.get("dsl_sha256") != current["dsl_sha256"]:
|
|
573
|
+
errors.append("recorded dsl_sha256 differs from the current DSL file (DSL edited after the run)")
|
|
574
|
+
if isinstance(approval, dict) and approval.get("approval_id") != current["dsl_approval"]["approval_id"]:
|
|
575
|
+
errors.append(f"recorded execution_plan approval {approval.get('approval_id')!r} is not the currently valid approval {current['dsl_approval']['approval_id']!r} (superseded or re-approved after the run)")
|
|
576
|
+
recorded_profile = entry.get("environment_profile")
|
|
577
|
+
current_profile = current["environment_profile"]
|
|
578
|
+
if not isinstance(recorded_profile, dict):
|
|
579
|
+
errors.append("environment_profile approval record is missing")
|
|
580
|
+
else:
|
|
581
|
+
if recorded_profile.get("name") != current_profile["name"] or recorded_profile.get("sha256") != current_profile["sha256"]:
|
|
582
|
+
errors.append("recorded environment profile hash differs from the current profile file (profile edited after the run)")
|
|
583
|
+
if recorded_profile.get("approval_id") != current_profile["approval_id"]:
|
|
584
|
+
errors.append(f"recorded environment_profile approval {recorded_profile.get('approval_id')!r} is not the currently valid approval {current_profile['approval_id']!r}")
|
|
585
|
+
errors.extend(playwright_adapter.recorded_run_errors(root, feature, tc_id, entry, run_dir_expired))
|
|
586
|
+
return errors
|
|
587
|
+
|
|
588
|
+
|
|
589
|
+
def manual_entry_errors(root: Path, feature: str, tc_id: str, source: dict[str, Any], entry: dict[str, Any], human: dict[str, Any] | None) -> list[str]:
|
|
590
|
+
errors: list[str] = []
|
|
591
|
+
if entry.get("origin") != "human_result_approval":
|
|
592
|
+
errors.append("origin must be human_result_approval")
|
|
593
|
+
if human is None:
|
|
594
|
+
return errors + ["no currently valid human_result approval backs this manual result"]
|
|
595
|
+
for key in ("status", "actual_result", "approval_id", "evidence_artifacts"):
|
|
596
|
+
if entry.get(key) != human[key]:
|
|
597
|
+
errors.append(f"recorded {key} does not match the current human_result approval (superseded or evidence changed)")
|
|
598
|
+
downgrade = entry.get("downgrade")
|
|
599
|
+
if source.get("automation_mode") == "automated":
|
|
600
|
+
override = valid_approval(root, feature, "mode_override", tc_id)
|
|
601
|
+
if not isinstance(downgrade, dict) or (downgrade.get("from"), downgrade.get("to")) != ("automated", "manual"):
|
|
602
|
+
errors.append("automated -> manual downgrade is not recorded")
|
|
603
|
+
elif not override["ok"] or downgrade.get("approval_id") != override["approval"]["id"]:
|
|
604
|
+
errors.append("automated -> manual downgrade has no currently valid mode_override approval matching the recorded approval_id")
|
|
605
|
+
return errors
|
|
606
|
+
|
|
607
|
+
|
|
608
|
+
def result_entry_errors(root: Path, feature: str, tc_id: str, source: dict[str, Any], suite: dict[str, Any], binding: dict[str, Any],
|
|
609
|
+
entry: dict[str, Any], human: dict[str, Any] | None, run_dir_expired: bool) -> list[str]:
|
|
610
|
+
"""Re-verify one recorded result against the CURRENT binding, frozen TC, approvals and evidence files."""
|
|
611
|
+
errors: list[str] = []
|
|
612
|
+
surface, _legacy = derive_surface(source)
|
|
613
|
+
mode = binding.get("mode")
|
|
614
|
+
if entry.get("mode") != mode:
|
|
615
|
+
errors.append(f"recorded mode {entry.get('mode')!r} does not match the current binding mode {mode!r}")
|
|
616
|
+
if entry.get("execution_surface") != surface:
|
|
617
|
+
errors.append(f"recorded execution_surface {entry.get('execution_surface')!r} does not match the frozen TC surface {surface!r}")
|
|
618
|
+
frozen_mode = source.get("automation_mode")
|
|
619
|
+
recorded_downgrade = entry.get("downgrade")
|
|
620
|
+
if mode != frozen_mode and (not isinstance(recorded_downgrade, dict) or (recorded_downgrade.get("from"), recorded_downgrade.get("to")) != (frozen_mode, mode)):
|
|
621
|
+
errors.append(f"downgrade {frozen_mode} -> {mode} is not recorded")
|
|
622
|
+
if mode == frozen_mode and recorded_downgrade is not None:
|
|
623
|
+
errors.append("a downgrade is recorded but the current binding does not downgrade this TC")
|
|
624
|
+
passed = entry.get("status") == "passed"
|
|
625
|
+
if passed and entry.get("tc_sha256") != tc_sha256(source):
|
|
626
|
+
errors.append("recorded tc_sha256 differs from the current frozen TC")
|
|
627
|
+
if mode == "manual":
|
|
628
|
+
return errors + manual_entry_errors(root, feature, tc_id, source, entry, human)
|
|
629
|
+
if mode != "automated":
|
|
630
|
+
return errors # blocked entries never pass
|
|
631
|
+
adapter = binding.get("adapter") if isinstance(binding.get("adapter"), dict) else {}
|
|
632
|
+
kind = adapter.get("kind")
|
|
633
|
+
if entry.get("adapter_kind") != kind:
|
|
634
|
+
errors.append(f"recorded adapter_kind {entry.get('adapter_kind')!r} does not match the current binding adapter {kind!r}")
|
|
635
|
+
return errors
|
|
636
|
+
if surface is not None and kind not in ADAPTERS_BY_SURFACE[surface]:
|
|
637
|
+
errors.append(f"execution_surface_mismatch: execution_surface={surface} cannot be bound to adapter kind {kind}")
|
|
638
|
+
if kind in RESULT_ORIGINS and entry.get("origin") != RESULT_ORIGINS[kind]:
|
|
639
|
+
errors.append(f"origin {entry.get('origin')!r} does not match adapter kind {kind} (expected {RESULT_ORIGINS[kind]})")
|
|
640
|
+
if kind == "command":
|
|
641
|
+
if entry.get("argv") != adapter.get("argv") or (valid_argv(adapter.get("argv")) and entry.get("command_hash") != digest(json.dumps(adapter["argv"], ensure_ascii=False, separators=(",", ":")))):
|
|
642
|
+
errors.append("recorded command argv/command_hash differs from the current binding")
|
|
643
|
+
elif kind == "http":
|
|
644
|
+
if entry.get("adapter_fingerprint") != digest(json.dumps(adapter, ensure_ascii=False, separators=(",", ":"))):
|
|
645
|
+
errors.append("recorded HTTP adapter fingerprint differs from the current binding")
|
|
646
|
+
elif kind == playwright_adapter.ADAPTER_KIND and passed:
|
|
647
|
+
errors.extend(playwright_entry_errors(root, feature, tc_id, source, suite, adapter, entry, run_dir_expired))
|
|
648
|
+
if passed and source.get("evidence_strategy") == "causal":
|
|
649
|
+
recorded = entry.get("evidence_artifacts")
|
|
650
|
+
current, current_errors = evidence_artifacts(root, binding.get("evidence_files"))
|
|
651
|
+
if not isinstance(recorded, list) or not recorded:
|
|
652
|
+
errors.append("causal evidence_artifacts are not recorded")
|
|
653
|
+
elif current_errors:
|
|
654
|
+
errors.extend(current_errors)
|
|
655
|
+
elif [{"path": item.get("path"), "sha256": item.get("sha256")} for item in recorded if isinstance(item, dict)] != [{"path": item["path"], "sha256": item["sha256"]} for item in current]:
|
|
656
|
+
errors.append("causal evidence files differ from the recorded hashes (evidence changed)")
|
|
657
|
+
return errors
|
|
658
|
+
|
|
659
|
+
|
|
660
|
+
def validate_results(root: Path, feature: str, today: date | None = None) -> dict[str, Any]:
|
|
661
|
+
"""Hard-gate check: every recorded result must still hold against the CURRENT frozen TCs, bindings, approvals and evidence."""
|
|
662
|
+
validation, suite, bindings = validate(root, feature)
|
|
417
663
|
errors = list(validation["errors"])
|
|
418
664
|
_, _, result_path = paths(root, feature)
|
|
419
665
|
results = read_json(result_path, errors, "acceptance-test results")
|
|
420
666
|
if suite is None or results is None:
|
|
421
|
-
return {"ok": False, "feature": feature, "errors": errors}
|
|
667
|
+
return {"ok": False, "feature": feature, "reason": "acceptance_results_not_passed", "errors": errors}
|
|
422
668
|
if results.get("origin") != "core_acceptance_test_runner" or results.get("feature") != feature:
|
|
423
669
|
errors.append("acceptance-test results identity is invalid")
|
|
424
670
|
meta = suite.get("view_meta") if isinstance(suite.get("view_meta"), dict) else {}
|
|
@@ -432,14 +678,97 @@ def validate_results(root: Path, feature: str) -> dict[str, Any]:
|
|
|
432
678
|
failed = sorted(item.get("tc_id", "<unknown>") for item in result_entries if not isinstance(item, dict) or item.get("status") != "passed")
|
|
433
679
|
if failed:
|
|
434
680
|
errors.append(f"frozen TC cases not passed: {', '.join(failed)}")
|
|
435
|
-
|
|
681
|
+
if suite is not None and isinstance(bindings, dict):
|
|
682
|
+
cases = frozen_cases(suite, [])
|
|
683
|
+
binding_by_id = {item["tc_id"]: item for item in bindings.get("cases", []) if isinstance(item, dict) and isinstance(item.get("tc_id"), str)}
|
|
684
|
+
_human_validation, human_results = validate_human_results(root, feature, suite, bindings)
|
|
685
|
+
expired = run_evidence_expired(root, feature, today)
|
|
686
|
+
seen: set[str] = set()
|
|
687
|
+
for entry in result_entries:
|
|
688
|
+
tc_id = entry.get("tc_id") if isinstance(entry, dict) else None
|
|
689
|
+
if not isinstance(tc_id, str) or tc_id not in cases or tc_id not in binding_by_id:
|
|
690
|
+
continue # coverage mismatch is already reported above
|
|
691
|
+
if tc_id in seen:
|
|
692
|
+
errors.append(f"{tc_id}: duplicate result entry")
|
|
693
|
+
continue
|
|
694
|
+
seen.add(tc_id)
|
|
695
|
+
errors.extend(f"{tc_id}: {error}" for error in result_entry_errors(root, feature, tc_id, cases[tc_id], suite, binding_by_id[tc_id], entry, human_results.get(tc_id), expired))
|
|
696
|
+
result = {"ok": not errors, "feature": feature, "errors": errors, "result_path": str(result_path.relative_to(root))}
|
|
697
|
+
if errors:
|
|
698
|
+
result["reason"] = "acceptance_results_not_passed"
|
|
699
|
+
return result
|
|
700
|
+
|
|
701
|
+
|
|
702
|
+
def closeout_completed_on(root: Path, feature: str) -> date | None:
|
|
703
|
+
"""Date release-closeout completed for this feature according to STATE.md, if it did."""
|
|
704
|
+
try:
|
|
705
|
+
text = (root / ".devflow/planning" / "STATE.md").read_text(encoding="utf-8")
|
|
706
|
+
except OSError:
|
|
707
|
+
return None
|
|
708
|
+
match = re.match(r"\A---\s*\n(.*?)\n---", text, re.S)
|
|
709
|
+
if not match:
|
|
710
|
+
return None
|
|
711
|
+
front = match.group(1)
|
|
712
|
+
field = lambda name: (re.search(rf"^{name}:\s*['\"]?([^'\"\n]*)['\"]?\s*$", front, re.M) or [None, None])[1]
|
|
713
|
+
completed = re.search(r"^completed_steps:\s*\n((?:\s*-\s*.*\n?)*)", front, re.M)
|
|
714
|
+
steps = {line.strip()[1:].strip().strip("'\"") for line in completed.group(1).splitlines()} if completed else set()
|
|
715
|
+
if field("current_feature") != feature or "release-closeout" not in steps:
|
|
716
|
+
return None
|
|
717
|
+
try:
|
|
718
|
+
return date.fromisoformat(str(field("state_updated_at")))
|
|
719
|
+
except ValueError:
|
|
720
|
+
return None
|
|
721
|
+
|
|
722
|
+
|
|
723
|
+
def referenced_execution_ids(result_path: Path) -> set[str]:
|
|
724
|
+
payload = read_json(result_path, [], "acceptance-test results") if result_path.is_file() else None
|
|
725
|
+
if not isinstance(payload, dict):
|
|
726
|
+
return set()
|
|
727
|
+
batches = [payload.get("results")] + [item.get("results") for item in payload.get("history", []) if isinstance(item, dict)]
|
|
728
|
+
return {entry["execution_id"] for batch in batches if isinstance(batch, list) for entry in batch
|
|
729
|
+
if isinstance(entry, dict) and isinstance(entry.get("execution_id"), str)}
|
|
730
|
+
|
|
731
|
+
|
|
732
|
+
def gc_runs(root: Path, feature: str, write: bool, retention_days: int = DEFAULT_RUN_RETENTION_DAYS,
|
|
733
|
+
unreferenced_ttl_days: int = DEFAULT_UNREFERENCED_RUN_TTL_DAYS, today: date | None = None) -> dict[str, Any]:
|
|
734
|
+
"""方案 §9.1: keep referenced runs until release-closeout + retention; expire unreferenced runs after a short TTL."""
|
|
735
|
+
today = today or datetime.now(timezone.utc).date()
|
|
736
|
+
directory = playwright_adapter.runs_dir(root, feature)
|
|
737
|
+
_, _, result_path = paths(root, feature)
|
|
738
|
+
referenced = referenced_execution_ids(result_path)
|
|
739
|
+
closed = closeout_completed_on(root, feature)
|
|
740
|
+
kept: list[dict[str, str]] = []
|
|
741
|
+
removed: list[str] = []
|
|
742
|
+
if directory.is_dir() and not directory.is_symlink():
|
|
743
|
+
for entry in sorted(directory.iterdir()):
|
|
744
|
+
name = entry.name
|
|
745
|
+
if entry.is_symlink() or not entry.is_dir() or not playwright_adapter.EXECUTION_ID_RE.fullmatch(name):
|
|
746
|
+
kept.append({"execution_id": name, "reason": "not a Core run directory"})
|
|
747
|
+
continue
|
|
748
|
+
started = datetime.strptime(name[:16], "%Y%m%dT%H%M%SZ").date()
|
|
749
|
+
if name in referenced:
|
|
750
|
+
if closed is None:
|
|
751
|
+
kept.append({"execution_id": name, "reason": "referenced by acceptance-results.json before release-closeout"})
|
|
752
|
+
continue
|
|
753
|
+
if today < closed + timedelta(days=retention_days):
|
|
754
|
+
kept.append({"execution_id": name, "reason": f"referenced; retained {retention_days} days after release-closeout"})
|
|
755
|
+
continue
|
|
756
|
+
elif today < started + timedelta(days=unreferenced_ttl_days):
|
|
757
|
+
kept.append({"execution_id": name, "reason": f"unreferenced run younger than {unreferenced_ttl_days} days"})
|
|
758
|
+
continue
|
|
759
|
+
removed.append(name)
|
|
760
|
+
if write:
|
|
761
|
+
shutil.rmtree(entry)
|
|
762
|
+
return {"ok": True, "feature": feature, "written": write, "removed": removed, "kept": kept}
|
|
436
763
|
|
|
437
764
|
|
|
438
765
|
def main() -> int:
|
|
439
766
|
parser = argparse.ArgumentParser()
|
|
440
|
-
parser.add_argument("action", choices=("validate", "run", "validate-results"))
|
|
767
|
+
parser.add_argument("action", choices=("validate", "validate-human", "run", "validate-results", "gc-runs"))
|
|
441
768
|
parser.add_argument("--feature", required=True)
|
|
442
769
|
parser.add_argument("--write", action="store_true")
|
|
770
|
+
parser.add_argument("--retention-days", type=int, default=DEFAULT_RUN_RETENTION_DAYS)
|
|
771
|
+
parser.add_argument("--unreferenced-ttl-days", type=int, default=DEFAULT_UNREFERENCED_RUN_TTL_DAYS)
|
|
443
772
|
parser.add_argument("--json", action="store_true")
|
|
444
773
|
args = parser.parse_args()
|
|
445
774
|
root = Path.cwd().resolve()
|
|
@@ -447,12 +776,19 @@ def main() -> int:
|
|
|
447
776
|
result = {"ok": False, "errors": ["invalid feature"]}
|
|
448
777
|
elif args.action == "validate":
|
|
449
778
|
result, _, _ = validate(root, args.feature)
|
|
779
|
+
elif args.action == "validate-human":
|
|
780
|
+
suite_path, _, _ = paths(root, args.feature)
|
|
781
|
+
errors: list[str] = []
|
|
782
|
+
suite = read_json(suite_path, errors, "TC suite")
|
|
783
|
+
result = {"ok": False, "feature": args.feature, "errors": errors} if suite is None else validate_human_results(root, args.feature, suite)[0]
|
|
450
784
|
elif args.action == "run":
|
|
451
785
|
result = run(root, args.feature, args.write) if args.write else {"ok": False, "errors": ["--write is required for run"]}
|
|
786
|
+
elif args.action == "gc-runs":
|
|
787
|
+
result = gc_runs(root, args.feature, args.write, max(args.retention_days, 0), max(args.unreferenced_ttl_days, 0))
|
|
452
788
|
else:
|
|
453
789
|
result = validate_results(root, args.feature)
|
|
454
790
|
print(json.dumps(result, ensure_ascii=False, indent=2))
|
|
455
|
-
return 0
|
|
791
|
+
return 0 if result.get("ok") is True else 1
|
|
456
792
|
|
|
457
793
|
|
|
458
794
|
if __name__ == "__main__":
|