its-magic 0.1.2-48 → 0.1.2-49
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/scripts/check_intake_template_parity.py +23 -0
- package/template/.cursor/commands/execute.md +49 -0
- package/template/.cursor/commands/qa.md +29 -0
- package/template/.cursor/commands/verify-work.md +28 -1
- package/template/.cursor/scratchpad.local.example.md +21 -0
- package/template/.cursor/scratchpad.md +20 -0
- package/template/docs/engineering/auto-orchestration-reference.md +18 -0
- package/template/docs/engineering/runbook.md +33 -0
- package/template/scripts/check_intake_template_parity.py +23 -0
- package/template/scripts/uat_probe_lib.py +576 -25
package/package.json
CHANGED
|
@@ -7,6 +7,7 @@ Scoped modes (DEC-0073 §10 / US-0090):
|
|
|
7
7
|
--scope=readme-feature-coverage DEC-0074 README feature-coverage pair table.
|
|
8
8
|
--scope=downstream-ci-guard DEC-0075 downstream CI guard script pair table.
|
|
9
9
|
--scope=us-0092 DEC-0078 full-autonomy outer driver + probe surfaces.
|
|
10
|
+
--scope=us-0093 DEC-0079 browser UAT probe surfaces.
|
|
10
11
|
--scope=all union of all tables.
|
|
11
12
|
"""
|
|
12
13
|
|
|
@@ -78,6 +79,26 @@ US0092_PAIRS: tuple[tuple[str, str], ...] = (
|
|
|
78
79
|
("docs/engineering/runbook.md", "template/docs/engineering/runbook.md"),
|
|
79
80
|
)
|
|
80
81
|
|
|
82
|
+
US0093_PAIRS: tuple[tuple[str, str], ...] = (
|
|
83
|
+
("scripts/uat_probe_lib.py", "template/scripts/uat_probe_lib.py"),
|
|
84
|
+
(".cursor/commands/verify-work.md", "template/.cursor/commands/verify-work.md"),
|
|
85
|
+
(".cursor/commands/qa.md", "template/.cursor/commands/qa.md"),
|
|
86
|
+
(".cursor/commands/execute.md", "template/.cursor/commands/execute.md"),
|
|
87
|
+
(
|
|
88
|
+
".cursor/scratchpad.local.example.md",
|
|
89
|
+
"template/.cursor/scratchpad.local.example.md",
|
|
90
|
+
),
|
|
91
|
+
("docs/engineering/runbook.md", "template/docs/engineering/runbook.md"),
|
|
92
|
+
(
|
|
93
|
+
"docs/engineering/auto-orchestration-reference.md",
|
|
94
|
+
"template/docs/engineering/auto-orchestration-reference.md",
|
|
95
|
+
),
|
|
96
|
+
(
|
|
97
|
+
"docs/engineering/context/installer-owned-paths.manifest",
|
|
98
|
+
"template/docs/engineering/context/installer-owned-paths.manifest",
|
|
99
|
+
),
|
|
100
|
+
)
|
|
101
|
+
|
|
81
102
|
DOWNSTREAM_CI_GUARD_PAIRS: tuple[tuple[str, str], ...] = (
|
|
82
103
|
(
|
|
83
104
|
"scripts/check_downstream_ci_guard.py",
|
|
@@ -95,12 +116,14 @@ SCOPES: dict[str, tuple[tuple[str, str], ...]] = {
|
|
|
95
116
|
"readme-feature-coverage": README_FEATURE_COVERAGE_PAIRS,
|
|
96
117
|
"downstream-ci-guard": DOWNSTREAM_CI_GUARD_PAIRS,
|
|
97
118
|
"us-0092": US0092_PAIRS,
|
|
119
|
+
"us-0093": US0093_PAIRS,
|
|
98
120
|
"all": (
|
|
99
121
|
INTAKE_TEMPLATE_PAIRS
|
|
100
122
|
+ CAVEMAN_COMPRESS_PAIRS
|
|
101
123
|
+ README_FEATURE_COVERAGE_PAIRS
|
|
102
124
|
+ DOWNSTREAM_CI_GUARD_PAIRS
|
|
103
125
|
+ US0092_PAIRS
|
|
126
|
+
+ US0093_PAIRS
|
|
104
127
|
),
|
|
105
128
|
}
|
|
106
129
|
|
|
@@ -48,6 +48,32 @@ as stale isolation evidence).
|
|
|
48
48
|
|
|
49
49
|
Release gate semantics (US-0039): mandatory gates (check-in test, QA, UAT) and no-bypass/override contract are enforced at `/release`; see `.cursor/commands/release.md` and `.cursor/commands/qa.md`.
|
|
50
50
|
|
|
51
|
+
### Browser UAT self-test (US-0093)
|
|
52
|
+
|
|
53
|
+
When **`UAT_BROWSER_PROBE_MODE=cursor`** (default) and acceptance steps classify as **`browser_smoke`**
|
|
54
|
+
(or automatable **`manual_operator`** re-routed per **DEC-0079** §4), execute the **Cursor browser MCP**
|
|
55
|
+
sequence during `/execute` when web UI verification applies (**BUG-0006**: **`scripts/uat_probe_lib.py`**
|
|
56
|
+
never invokes MCP — subagent only):
|
|
57
|
+
|
|
58
|
+
1. **Resolve target URL** — `docs/engineering/runtime-connectivity.md` first `http(s)://`; else
|
|
59
|
+
`package.json` dev/start + **`DEV_SERVER_PORT`** scratchpad override.
|
|
60
|
+
2. **`browser_navigate`** — load URL; respect enterprise origin allowlist.
|
|
61
|
+
3. **Step plan** — map automatable verbs to **`browser_click`** / **`browser_type`** / **`browser_scroll`**;
|
|
62
|
+
**never** fill password/credential fields; **never** read **`.env`** paths.
|
|
63
|
+
4. **`browser_screenshot`** — write to **`sprints/Sxxxx/evidence/browser/<probe_id>-<seq>.png`** (max **5**).
|
|
64
|
+
5. **Console + network evidence** — counts + summary path refs only (no inline secrets).
|
|
65
|
+
6. **Verdict** — set **`passed`**, **`reason_code`**, **`browser_evidence_refs`** on the matching
|
|
66
|
+
**`probe_results[]`** row in **`uat.json`**. **`passed=true`** in **`cursor`** mode requires
|
|
67
|
+
non-empty **`navigation_url`** + at least one screenshot or console/network summary path — else
|
|
68
|
+
downgrade to **`UAT_BROWSER_PROBE_FAILED`**.
|
|
69
|
+
7. **Fallback trigger** — MCP unavailable → record **`UAT_BROWSER_UNAVAILABLE`**; when
|
|
70
|
+
**`UAT_BROWSER_FALLBACK_CHAIN=1`**, stdlib HTTP/Playwright fallback runs via **`uat_probe_lib.py`**.
|
|
71
|
+
Both fail → **`UAT_BROWSER_PROBE_FAILED`** (no silent PASS).
|
|
72
|
+
|
|
73
|
+
Optional validation: **`python scripts/uat_probe_lib.py --merge-result <fragment.json>`** —
|
|
74
|
+
evidence-required-on-PASS in **`cursor`** mode. No auto-read **`.env`**, no credential auto-fill,
|
|
75
|
+
no intake evidence mutation.
|
|
76
|
+
|
|
51
77
|
## Intake evidence tooling reference (US-0078 / DEC-0060)
|
|
52
78
|
|
|
53
79
|
Stories that harden intake persistence ship **`scripts/intake_evidence_lib.py`**,
|
|
@@ -77,6 +103,29 @@ parity for listed paths: **`python scripts/check_token_cost_parity.py --repo .`*
|
|
|
77
103
|
- `/execute` must not start/continue implementation solely based on
|
|
78
104
|
non-canonical status evidence.
|
|
79
105
|
|
|
106
|
+
## Browser UAT self-test (US-0093)
|
|
107
|
+
|
|
108
|
+
When acceptance steps classify as **`browser_smoke`** (or automatable **`manual_operator`**
|
|
109
|
+
reclassified per **DEC-0079** §4) and **`UAT_BROWSER_PROBE_MODE=cursor`**, execute the **Cursor
|
|
110
|
+
browser MCP** sequence during execute-phase verification — **lib never calls MCP directly**
|
|
111
|
+
(**BUG-0006**):
|
|
112
|
+
|
|
113
|
+
1. **Resolve target URL** — `docs/engineering/runtime-connectivity.md` first `http(s)://`; else
|
|
114
|
+
dev-server port from `package.json` + scratchpad **`DEV_SERVER_PORT`**.
|
|
115
|
+
2. **`browser_navigate`** — respect origin allowlist.
|
|
116
|
+
3. **Step plan** — **`browser_click`** / **`browser_type`** / **`browser_scroll`** when
|
|
117
|
+
automatable; **never** fill password/credential fields or read **`.env`**.
|
|
118
|
+
4. **`browser_screenshot`** → **`sprints/Sxxxx/evidence/browser/<probe_id>-<seq>.png`** (max **5**).
|
|
119
|
+
5. Console + network summary path refs only (no inline secrets).
|
|
120
|
+
6. Write **`browser_evidence_refs`** to **`uat.json`** `probe_results[]`; mirror
|
|
121
|
+
**Runtime browser evidence** in **`qa-findings.md`**.
|
|
122
|
+
7. MCP unavailable → **`UAT_BROWSER_UNAVAILABLE`** + stdlib fallback when
|
|
123
|
+
**`UAT_BROWSER_FALLBACK_CHAIN=1`**; validate with
|
|
124
|
+
**`python scripts/uat_probe_lib.py --merge-result <fragment.json>`**.
|
|
125
|
+
|
|
126
|
+
Reason codes: **`UAT_BROWSER_UNAVAILABLE`**, **`UAT_BROWSER_PROBE_FAILED`**,
|
|
127
|
+
**`UAT_BROWSER_PROBE_TIMEOUT`**. **No silent PASS** without agent evidence in **`cursor`** mode.
|
|
128
|
+
|
|
80
129
|
## Steps
|
|
81
130
|
1. Implement one task at a time.
|
|
82
131
|
2. Update summary and engineering state.
|
|
@@ -56,6 +56,35 @@ acceptance steps, execute probes where stack profile resolves, record evidence i
|
|
|
56
56
|
Fail closed with **`UAT_PROBE_UNRESOLVED`** (not PASS) when no probe maps.
|
|
57
57
|
Forbidden: auto-read **`.env`**, mutate intake evidence — **`UAT_PROBE_FORBIDDEN`**.
|
|
58
58
|
|
|
59
|
+
Reason codes extend with **`UAT_BROWSER_UNAVAILABLE`**, **`UAT_BROWSER_PROBE_FAILED`**,
|
|
60
|
+
**`UAT_BROWSER_PROBE_TIMEOUT`** (**US-0093** / **DEC-0079**).
|
|
61
|
+
|
|
62
|
+
### Browser UAT self-test (US-0093)
|
|
63
|
+
|
|
64
|
+
When **`UAT_BROWSER_PROBE_MODE=cursor`** (default) and step classifies as **`browser_smoke`**
|
|
65
|
+
(or automatable **`manual_operator`** reclassified per **DEC-0079** §4), execute the **Cursor
|
|
66
|
+
browser MCP** sequence — **lib never calls MCP directly** (**BUG-0006**):
|
|
67
|
+
|
|
68
|
+
1. **Resolve target URL** — `docs/engineering/runtime-connectivity.md` first `http(s)://`; else
|
|
69
|
+
dev-server port from `package.json` + scratchpad **`DEV_SERVER_PORT`**.
|
|
70
|
+
2. **`browser_navigate`** — load health/app URL; respect enterprise origin allowlist.
|
|
71
|
+
3. **Step plan** — map acceptance verbs to **`browser_click`** / **`browser_type`** /
|
|
72
|
+
**`browser_scroll`** when automatable; **never** fill password/credential fields or read **`.env`**.
|
|
73
|
+
4. **`browser_screenshot`** — write to **`sprints/Sxxxx/evidence/browser/<probe_id>-<seq>.png`**
|
|
74
|
+
(max **5** per probe).
|
|
75
|
+
5. **Console + network evidence** — counts + summary path refs only (no inline secrets).
|
|
76
|
+
6. **Verdict** — set **`passed`**, **`reason_code`**, **`browser_evidence_refs`** in
|
|
77
|
+
**`uat.json`** `probe_results[]`; **`passed=true`** in **`cursor`** mode **requires** non-empty
|
|
78
|
+
**`navigation_url`** + at least one screenshot or console/network summary path — else
|
|
79
|
+
**`UAT_BROWSER_PROBE_FAILED`**. Mirror under **`qa-findings.md`** **Runtime browser evidence**.
|
|
80
|
+
7. **Fallback** — MCP unavailable → record **`UAT_BROWSER_UNAVAILABLE`**, run stdlib fallback when
|
|
81
|
+
**`UAT_BROWSER_FALLBACK_CHAIN=1`**; validate via
|
|
82
|
+
**`python scripts/uat_probe_lib.py --merge-result <fragment.json>`**.
|
|
83
|
+
|
|
84
|
+
**No silent PASS** — stdlib alone does **not** PASS **`browser_smoke`** in **`cursor`** mode
|
|
85
|
+
without agent **`browser_evidence_refs`**. Security: no auto-read **`.env`**, no credential
|
|
86
|
+
auto-fill, no intake evidence mutation — **`UAT_PROBE_FORBIDDEN`** unchanged.
|
|
87
|
+
|
|
59
88
|
## Steps
|
|
60
89
|
0. If `SECURITY_REVIEW=1`, verify `docs/engineering/security-review.md` exists
|
|
61
90
|
and has no unresolved `critical` findings before proceeding. If unresolved
|
|
@@ -103,12 +103,39 @@ Record evidence in **`sprints/Sxxxx/uat.json`** `probe_results[]` and
|
|
|
103
103
|
**No silent PASS** — unresolvable steps must record fail-closed reason codes:
|
|
104
104
|
|
|
105
105
|
`UAT_PROBE_UNRESOLVED`, `UAT_STACK_PROFILE_UNKNOWN`, `UAT_PROBE_TIMEOUT`,
|
|
106
|
-
`UAT_PROBE_FAILED`, `UAT_PROBE_FORBIDDEN`, `UAT_PROBE_PASS
|
|
106
|
+
`UAT_PROBE_FAILED`, `UAT_PROBE_FORBIDDEN`, `UAT_PROBE_PASS`,
|
|
107
|
+
`UAT_BROWSER_UNAVAILABLE`, `UAT_BROWSER_PROBE_FAILED`, `UAT_BROWSER_PROBE_TIMEOUT`.
|
|
107
108
|
|
|
108
109
|
Probe catalog: **`build`**, **`test`**, **`api_health`**, **`process_health`**,
|
|
109
110
|
**`browser_smoke`**, **`cli_smoke`**, **`manual_operator`**. CLI:
|
|
110
111
|
`python scripts/uat_probe_lib.py --repo . --step "<acceptance step>" --report`.
|
|
111
112
|
|
|
113
|
+
### Browser UAT self-test (US-0093)
|
|
114
|
+
|
|
115
|
+
When **`UAT_BROWSER_PROBE_MODE=cursor`** (default) and step classifies as **`browser_smoke`**
|
|
116
|
+
(or automatable **`manual_operator`** reclassified per **DEC-0079** §4), execute the **Cursor
|
|
117
|
+
browser MCP** sequence — **lib never calls MCP directly** (**BUG-0006**):
|
|
118
|
+
|
|
119
|
+
1. **Resolve target URL** — `docs/engineering/runtime-connectivity.md` first `http(s)://`; else
|
|
120
|
+
dev-server port from `package.json` + scratchpad **`DEV_SERVER_PORT`**.
|
|
121
|
+
2. **`browser_navigate`** — load health/app URL; respect enterprise origin allowlist.
|
|
122
|
+
3. **Step plan** — map acceptance verbs to **`browser_click`** / **`browser_type`** /
|
|
123
|
+
**`browser_scroll`** when automatable; **never** fill password/credential fields or read **`.env`**.
|
|
124
|
+
4. **`browser_screenshot`** — write to **`sprints/Sxxxx/evidence/browser/<probe_id>-<seq>.png`**
|
|
125
|
+
(max **5** per probe).
|
|
126
|
+
5. **Console + network evidence** — counts + summary path refs only (no inline secrets).
|
|
127
|
+
6. **Verdict** — set **`passed`**, **`reason_code`**, **`browser_evidence_refs`** in
|
|
128
|
+
**`uat.json`** `probe_results[]`; **`passed=true`** in **`cursor`** mode **requires** non-empty
|
|
129
|
+
**`navigation_url`** + at least one screenshot or console/network summary path — else
|
|
130
|
+
**`UAT_BROWSER_PROBE_FAILED`**. Mirror under **`qa-findings.md`** **Runtime browser evidence**.
|
|
131
|
+
7. **Fallback** — MCP unavailable → record **`UAT_BROWSER_UNAVAILABLE`**, run stdlib fallback when
|
|
132
|
+
**`UAT_BROWSER_FALLBACK_CHAIN=1`**; validate via
|
|
133
|
+
**`python scripts/uat_probe_lib.py --merge-result <fragment.json>`**.
|
|
134
|
+
|
|
135
|
+
**No silent PASS** — stdlib alone does **not** PASS **`browser_smoke`** in **`cursor`** mode
|
|
136
|
+
without agent **`browser_evidence_refs`**. Security: no auto-read **`.env`**, no credential
|
|
137
|
+
auto-fill, no intake evidence mutation — **`UAT_PROBE_FORBIDDEN`** unchanged.
|
|
138
|
+
|
|
112
139
|
## Steps
|
|
113
140
|
1. Convert acceptance criteria into testable UAT steps. Derive steps directly from the story's acceptance criteria in `docs/product/acceptance.md`. Each AC should map to at least one UAT step. Run **`scripts/uat_probe_lib.py`** for each step where automation applies; record `probe_results[]` in **`uat.json`**.
|
|
114
141
|
2. Populate UAT artifacts: write derived steps into `uat.json` (with description and result per step, accurate pass/fail counts) and `uat.md` (step list with results, summary section). Ensure UAT artifacts are in **populated** state per DEC-0009 — not placeholder.
|
|
@@ -238,6 +238,27 @@ DOC_DETAIL_LEVEL=balanced
|
|
|
238
238
|
# - README_FEATURE_COVERAGE_ENFORCE: 0|1 (default 0 until backfill + --report green)
|
|
239
239
|
README_FEATURE_COVERAGE_ENFORCE=1
|
|
240
240
|
|
|
241
|
+
#
|
|
242
|
+
# ## Browser UAT self-test (US-0093 / DEC-0079)
|
|
243
|
+
# Two-tier browser probe: stdlib lib classifies + agent owns Cursor browser MCP (BUG-0006).
|
|
244
|
+
# - UAT_BROWSER_PROBE_MODE: cursor|http_fallback|playwright_fallback (default cursor)
|
|
245
|
+
# - cursor: agent executes MCP sequence; lib emits plan + UAT_PROBE_UNRESOLVED until evidence
|
|
246
|
+
# - http_fallback: stdlib HTTP GET (CI recipe — set this in CI)
|
|
247
|
+
# - playwright_fallback: subprocess Playwright primary; HTTP fallback when missing
|
|
248
|
+
# - UAT_BROWSER_FALLBACK_CHAIN: 0|1 (default 1; enable HTTP → Playwright after MCP unavailable)
|
|
249
|
+
# - UAT_PROCESS_HEALTH_POLL_SECONDS: positive int (default 60; process_health readiness cap)
|
|
250
|
+
# - UAT_PROCESS_HEALTH_POLL_INTERVAL_SECONDS: positive int (default 2; poll interval)
|
|
251
|
+
# - DEV_SERVER_PORT: int (optional; URL/port inference override)
|
|
252
|
+
# - DEV_SERVER_COMMAND: shell command (optional; process_health startup override)
|
|
253
|
+
# Interaction: orthogonal to PERMISSION_MODE and Cursor browser approval modes (manual / allow-list /
|
|
254
|
+
# auto-run per vendor docs). Health URLs from docs/engineering/runtime-connectivity.md first.
|
|
255
|
+
UAT_BROWSER_PROBE_MODE=cursor
|
|
256
|
+
UAT_BROWSER_FALLBACK_CHAIN=1
|
|
257
|
+
UAT_PROCESS_HEALTH_POLL_SECONDS=60
|
|
258
|
+
UAT_PROCESS_HEALTH_POLL_INTERVAL_SECONDS=2
|
|
259
|
+
DEV_SERVER_PORT=
|
|
260
|
+
DEV_SERVER_COMMAND=
|
|
261
|
+
|
|
241
262
|
#
|
|
242
263
|
# ## Caveman mode (US-0089)
|
|
243
264
|
# Response-side voice toggle. Default off. Composition is orthogonal to
|
|
@@ -240,3 +240,23 @@ USER_GUIDE_MODE=0
|
|
|
240
240
|
DOC_AUDIENCE_PROFILE=both
|
|
241
241
|
DOC_DETAIL_LEVEL=balanced
|
|
242
242
|
|
|
243
|
+
#
|
|
244
|
+
# ## Browser UAT self-test (US-0093 / DEC-0079)
|
|
245
|
+
# Two-tier browser probe: stdlib lib classifies + agent owns Cursor browser MCP (BUG-0006).
|
|
246
|
+
# - UAT_BROWSER_PROBE_MODE: cursor|http_fallback|playwright_fallback (default cursor)
|
|
247
|
+
# - cursor: agent executes MCP sequence; lib emits plan + UAT_PROBE_UNRESOLVED until evidence
|
|
248
|
+
# - http_fallback: stdlib HTTP GET (CI recipe — set this in CI)
|
|
249
|
+
# - playwright_fallback: subprocess Playwright primary; HTTP fallback when missing
|
|
250
|
+
# - UAT_BROWSER_FALLBACK_CHAIN: 0|1 (default 1; enable HTTP → Playwright after MCP unavailable)
|
|
251
|
+
# - UAT_PROCESS_HEALTH_POLL_SECONDS: positive int (default 60; process_health readiness cap)
|
|
252
|
+
# - UAT_PROCESS_HEALTH_POLL_INTERVAL_SECONDS: positive int (default 2; poll interval)
|
|
253
|
+
# - DEV_SERVER_PORT: int (optional; URL/port inference override)
|
|
254
|
+
# - DEV_SERVER_COMMAND: shell command (optional; process_health startup override)
|
|
255
|
+
# Interaction: orthogonal to PERMISSION_MODE and Cursor browser approval modes (manual / allow-list /
|
|
256
|
+
# auto-run per vendor docs). Health URLs from docs/engineering/runtime-connectivity.md first.
|
|
257
|
+
UAT_BROWSER_PROBE_MODE=cursor
|
|
258
|
+
UAT_BROWSER_FALLBACK_CHAIN=1
|
|
259
|
+
UAT_PROCESS_HEALTH_POLL_SECONDS=60
|
|
260
|
+
UAT_PROCESS_HEALTH_POLL_INTERVAL_SECONDS=2
|
|
261
|
+
DEV_SERVER_PORT=
|
|
262
|
+
DEV_SERVER_COMMAND=
|
|
@@ -815,6 +815,24 @@ Cap exhaustion → exit **6** `BLOCK_RETRY_CAP_EXHAUSTED`. Ordering: outer drive
|
|
|
815
815
|
`AUTO_LOOP_MAX_CYCLES` first; orchestrator checks `AUTO_IMPLEMENTATION_LOOP` +
|
|
816
816
|
`AUTO_BLOCK_RETRY_MAX` before scheduling remediation.
|
|
817
817
|
|
|
818
|
+
### Browser UAT self-test (US-0093)
|
|
819
|
+
|
|
820
|
+
Two-tier browser UAT: **`scripts/uat_probe_lib.py`** classifies and completes subprocess probes;
|
|
821
|
+
phase subagents (**`/verify-work`**, **`/qa`**, **`/execute`**) own Cursor browser MCP when
|
|
822
|
+
**`UAT_BROWSER_PROBE_MODE=cursor`** (default). Lib **never** invokes browser MCP (**BUG-0006**).
|
|
823
|
+
|
|
824
|
+
| Key | Role |
|
|
825
|
+
|-----|------|
|
|
826
|
+
| `UAT_BROWSER_PROBE_MODE` | `cursor` \| `http_fallback` \| `playwright_fallback` |
|
|
827
|
+
| `UAT_BROWSER_FALLBACK_CHAIN` | HTTP → Playwright after **`UAT_BROWSER_UNAVAILABLE`** |
|
|
828
|
+
| `UAT_PROCESS_HEALTH_POLL_SECONDS` / `UAT_PROCESS_HEALTH_POLL_INTERVAL_SECONDS` | `process_health` readiness |
|
|
829
|
+
| `DEV_SERVER_PORT` / `DEV_SERVER_COMMAND` | Dev-server inference overrides |
|
|
830
|
+
|
|
831
|
+
CI: set **`UAT_BROWSER_PROBE_MODE=http_fallback`**. Evidence:
|
|
832
|
+
**`sprints/Sxxxx/evidence/browser/`** + **`browser_evidence_refs`** in **`uat.json`**.
|
|
833
|
+
Validate: **`python scripts/uat_probe_lib.py --merge-result <fragment.json>`**.
|
|
834
|
+
Manual override: **`@browser`** in Agent panel. Orthogonal to **`PERMISSION_MODE`** and browser
|
|
835
|
+
approval modes.
|
|
818
836
|
### `AUTO_QUIET` vs `TOKEN_PROFILE` (US-0088 / AC-2)
|
|
819
837
|
|
|
820
838
|
| Key | Values | Role |
|
|
@@ -1406,6 +1406,39 @@ without side effects. `--invoke-cmd` overrides the default normative `/auto …`
|
|
|
1406
1406
|
|
|
1407
1407
|
UAT self-verify: **`scripts/uat_probe_lib.py`** shared by **`/verify-work`** and **`/qa`**.
|
|
1408
1408
|
|
|
1409
|
+
### Browser UAT self-test (US-0093)
|
|
1410
|
+
|
|
1411
|
+
Enable Cursor browser-integrated UAT probes for web acceptance steps (**DEC-0079**).
|
|
1412
|
+
|
|
1413
|
+
#### Scratchpad keys
|
|
1414
|
+
|
|
1415
|
+
| Key | Values | Default | Notes |
|
|
1416
|
+
|-----|--------|---------|-------|
|
|
1417
|
+
| `UAT_BROWSER_PROBE_MODE` | `cursor` \| `http_fallback` \| `playwright_fallback` | `cursor` | Primary probe path |
|
|
1418
|
+
| `UAT_BROWSER_FALLBACK_CHAIN` | `0` \| `1` | `1` | HTTP → Playwright after MCP unavailable |
|
|
1419
|
+
| `UAT_PROCESS_HEALTH_POLL_SECONDS` | int | `60` | Readiness poll cap |
|
|
1420
|
+
| `UAT_PROCESS_HEALTH_POLL_INTERVAL_SECONDS` | int | `2` | Poll interval |
|
|
1421
|
+
| `DEV_SERVER_PORT` | int | unset | Port override |
|
|
1422
|
+
| `DEV_SERVER_COMMAND` | command | unset | Startup override |
|
|
1423
|
+
|
|
1424
|
+
Orthogonal to **`PERMISSION_MODE`** and Cursor browser approval modes. Health URLs from
|
|
1425
|
+
**`docs/engineering/runtime-connectivity.md`** first.
|
|
1426
|
+
|
|
1427
|
+
#### CI recipe
|
|
1428
|
+
|
|
1429
|
+
Set **`UAT_BROWSER_PROBE_MODE=http_fallback`** in CI — never false PASS without agent evidence.
|
|
1430
|
+
|
|
1431
|
+
#### Evidence layout
|
|
1432
|
+
|
|
1433
|
+
Binary artifacts under **`sprints/Sxxxx/evidence/browser/`** (gitignored OK). JSON carries path
|
|
1434
|
+
refs only. Validate agent write-back:
|
|
1435
|
+
`python scripts/uat_probe_lib.py --merge-result sprints/Sxxxx/evidence/browser/fragment.json`.
|
|
1436
|
+
|
|
1437
|
+
#### Manual override
|
|
1438
|
+
|
|
1439
|
+
Use **`@browser`** in Agent panel or invoke browser tools manually when MCP sequence needs operator
|
|
1440
|
+
approval for production-like targets.
|
|
1441
|
+
|
|
1409
1442
|
### Caveman mode (US-0089)
|
|
1410
1443
|
|
|
1411
1444
|
Optional response-side terse / imperative assistant voice. **Default off.**
|
|
@@ -7,6 +7,7 @@ Scoped modes (DEC-0073 §10 / US-0090):
|
|
|
7
7
|
--scope=readme-feature-coverage DEC-0074 README feature-coverage pair table.
|
|
8
8
|
--scope=downstream-ci-guard DEC-0075 downstream CI guard script pair table.
|
|
9
9
|
--scope=us-0092 DEC-0078 full-autonomy outer driver + probe surfaces.
|
|
10
|
+
--scope=us-0093 DEC-0079 browser UAT probe surfaces.
|
|
10
11
|
--scope=all union of all tables.
|
|
11
12
|
"""
|
|
12
13
|
|
|
@@ -78,6 +79,26 @@ US0092_PAIRS: tuple[tuple[str, str], ...] = (
|
|
|
78
79
|
("docs/engineering/runbook.md", "template/docs/engineering/runbook.md"),
|
|
79
80
|
)
|
|
80
81
|
|
|
82
|
+
US0093_PAIRS: tuple[tuple[str, str], ...] = (
|
|
83
|
+
("scripts/uat_probe_lib.py", "template/scripts/uat_probe_lib.py"),
|
|
84
|
+
(".cursor/commands/verify-work.md", "template/.cursor/commands/verify-work.md"),
|
|
85
|
+
(".cursor/commands/qa.md", "template/.cursor/commands/qa.md"),
|
|
86
|
+
(".cursor/commands/execute.md", "template/.cursor/commands/execute.md"),
|
|
87
|
+
(
|
|
88
|
+
".cursor/scratchpad.local.example.md",
|
|
89
|
+
"template/.cursor/scratchpad.local.example.md",
|
|
90
|
+
),
|
|
91
|
+
("docs/engineering/runbook.md", "template/docs/engineering/runbook.md"),
|
|
92
|
+
(
|
|
93
|
+
"docs/engineering/auto-orchestration-reference.md",
|
|
94
|
+
"template/docs/engineering/auto-orchestration-reference.md",
|
|
95
|
+
),
|
|
96
|
+
(
|
|
97
|
+
"docs/engineering/context/installer-owned-paths.manifest",
|
|
98
|
+
"template/docs/engineering/context/installer-owned-paths.manifest",
|
|
99
|
+
),
|
|
100
|
+
)
|
|
101
|
+
|
|
81
102
|
DOWNSTREAM_CI_GUARD_PAIRS: tuple[tuple[str, str], ...] = (
|
|
82
103
|
(
|
|
83
104
|
"scripts/check_downstream_ci_guard.py",
|
|
@@ -95,12 +116,14 @@ SCOPES: dict[str, tuple[tuple[str, str], ...]] = {
|
|
|
95
116
|
"readme-feature-coverage": README_FEATURE_COVERAGE_PAIRS,
|
|
96
117
|
"downstream-ci-guard": DOWNSTREAM_CI_GUARD_PAIRS,
|
|
97
118
|
"us-0092": US0092_PAIRS,
|
|
119
|
+
"us-0093": US0093_PAIRS,
|
|
98
120
|
"all": (
|
|
99
121
|
INTAKE_TEMPLATE_PAIRS
|
|
100
122
|
+ CAVEMAN_COMPRESS_PAIRS
|
|
101
123
|
+ README_FEATURE_COVERAGE_PAIRS
|
|
102
124
|
+ DOWNSTREAM_CI_GUARD_PAIRS
|
|
103
125
|
+ US0092_PAIRS
|
|
126
|
+
+ US0093_PAIRS
|
|
104
127
|
),
|
|
105
128
|
}
|
|
106
129
|
|
|
@@ -1,17 +1,19 @@
|
|
|
1
1
|
#!/usr/bin/env python3
|
|
2
2
|
"""
|
|
3
|
-
UAT probe resolver (US-0092 / DEC-0078).
|
|
3
|
+
UAT probe resolver (US-0092 / DEC-0078; US-0093 / DEC-0079).
|
|
4
4
|
|
|
5
5
|
Shared by /verify-work and /qa for self-verify acceptance steps.
|
|
6
|
-
Fail-closed — no silent PASS.
|
|
6
|
+
Fail-closed — no silent PASS. Lib never invokes browser MCP (BUG-0006).
|
|
7
7
|
"""
|
|
8
8
|
|
|
9
9
|
from __future__ import annotations
|
|
10
10
|
|
|
11
11
|
import json
|
|
12
|
+
import os
|
|
12
13
|
import re
|
|
13
14
|
import subprocess
|
|
14
15
|
import sys
|
|
16
|
+
import time
|
|
15
17
|
import urllib.error
|
|
16
18
|
import urllib.request
|
|
17
19
|
from pathlib import Path
|
|
@@ -22,6 +24,9 @@ UAT_PROBE_TIMEOUT = "UAT_PROBE_TIMEOUT"
|
|
|
22
24
|
UAT_PROBE_FAILED = "UAT_PROBE_FAILED"
|
|
23
25
|
UAT_PROBE_FORBIDDEN = "UAT_PROBE_FORBIDDEN"
|
|
24
26
|
UAT_PROBE_PASS = "UAT_PROBE_PASS"
|
|
27
|
+
UAT_BROWSER_UNAVAILABLE = "UAT_BROWSER_UNAVAILABLE"
|
|
28
|
+
UAT_BROWSER_PROBE_FAILED = "UAT_BROWSER_PROBE_FAILED"
|
|
29
|
+
UAT_BROWSER_PROBE_TIMEOUT = "UAT_BROWSER_PROBE_TIMEOUT"
|
|
25
30
|
|
|
26
31
|
PROBE_KINDS = (
|
|
27
32
|
"build",
|
|
@@ -34,8 +39,43 @@ PROBE_KINDS = (
|
|
|
34
39
|
)
|
|
35
40
|
|
|
36
41
|
FORBIDDEN_PATH_TOKENS = (".env", "intake_evidence", "handoffs/intake_evidence")
|
|
42
|
+
SECRET_FORBIDDEN_TOKENS = ("password", "credential", "api key")
|
|
43
|
+
|
|
44
|
+
JUDGMENT_DENY_TOKENS = (
|
|
45
|
+
"visually",
|
|
46
|
+
"aesthetically",
|
|
47
|
+
"operator confirms",
|
|
48
|
+
"subjective",
|
|
49
|
+
"human judgment",
|
|
50
|
+
"eyeball",
|
|
51
|
+
"manually verify appearance",
|
|
52
|
+
"approve layout",
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
AUTOMATABLE_UI_TOKENS = (
|
|
56
|
+
"click",
|
|
57
|
+
"fill",
|
|
58
|
+
"navigate",
|
|
59
|
+
"smoke",
|
|
60
|
+
"form",
|
|
61
|
+
"submit",
|
|
62
|
+
"button",
|
|
63
|
+
"page load",
|
|
64
|
+
"scroll",
|
|
65
|
+
"type into",
|
|
66
|
+
"select",
|
|
67
|
+
"checkbox",
|
|
68
|
+
"dropdown",
|
|
69
|
+
"ui",
|
|
70
|
+
"browser",
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
BROWSER_PROBE_MODES = ("cursor", "http_fallback", "playwright_fallback")
|
|
37
74
|
|
|
38
75
|
DEFAULT_PROBE_TIMEOUT = 120
|
|
76
|
+
DEFAULT_POLL_SECONDS = 60
|
|
77
|
+
DEFAULT_POLL_INTERVAL = 2
|
|
78
|
+
MAX_SCREENSHOTS = 5
|
|
39
79
|
|
|
40
80
|
|
|
41
81
|
def _merge_scratchpad(repo: Path) -> dict[str, str]:
|
|
@@ -54,6 +94,39 @@ def _merge_scratchpad(repo: Path) -> dict[str, str]:
|
|
|
54
94
|
return values
|
|
55
95
|
|
|
56
96
|
|
|
97
|
+
def _read_int(merged: dict[str, str], key: str, default: int) -> int:
|
|
98
|
+
raw = merged.get(key, "")
|
|
99
|
+
if not raw:
|
|
100
|
+
return default
|
|
101
|
+
try:
|
|
102
|
+
return max(1, int(raw))
|
|
103
|
+
except ValueError:
|
|
104
|
+
return default
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def read_browser_probe_mode(repo: Path) -> str:
|
|
108
|
+
merged = _merge_scratchpad(repo)
|
|
109
|
+
mode = merged.get("UAT_BROWSER_PROBE_MODE", "cursor").strip()
|
|
110
|
+
return mode if mode in BROWSER_PROBE_MODES else "cursor"
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def read_fallback_chain(repo: Path) -> bool:
|
|
114
|
+
merged = _merge_scratchpad(repo)
|
|
115
|
+
raw = merged.get("UAT_BROWSER_FALLBACK_CHAIN", "1").strip()
|
|
116
|
+
if raw in ("0", "1"):
|
|
117
|
+
return raw == "1"
|
|
118
|
+
return True
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def read_poll_settings(repo: Path) -> tuple[int, int]:
|
|
122
|
+
merged = _merge_scratchpad(repo)
|
|
123
|
+
cap = _read_int(merged, "UAT_PROCESS_HEALTH_POLL_SECONDS", DEFAULT_POLL_SECONDS)
|
|
124
|
+
interval = _read_int(
|
|
125
|
+
merged, "UAT_PROCESS_HEALTH_POLL_INTERVAL_SECONDS", DEFAULT_POLL_INTERVAL
|
|
126
|
+
)
|
|
127
|
+
return cap, interval
|
|
128
|
+
|
|
129
|
+
|
|
57
130
|
def detect_stack_profile(repo: Path) -> str | None:
|
|
58
131
|
if (repo / "package.json").is_file():
|
|
59
132
|
return "node"
|
|
@@ -73,7 +146,17 @@ def detect_stack_profile(repo: Path) -> str | None:
|
|
|
73
146
|
|
|
74
147
|
def _forbidden(step_text: str) -> bool:
|
|
75
148
|
lower = step_text.lower()
|
|
76
|
-
|
|
149
|
+
if any(tok in lower for tok in FORBIDDEN_PATH_TOKENS):
|
|
150
|
+
return True
|
|
151
|
+
return any(tok in lower for tok in SECRET_FORBIDDEN_TOKENS)
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def _has_judgment_deny(lower: str) -> bool:
|
|
155
|
+
return any(tok in lower for tok in JUDGMENT_DENY_TOKENS)
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def _has_automatable_ui(lower: str) -> bool:
|
|
159
|
+
return any(tok in lower for tok in AUTOMATABLE_UI_TOKENS)
|
|
77
160
|
|
|
78
161
|
|
|
79
162
|
def _read_test_command(repo: Path) -> str | None:
|
|
@@ -111,13 +194,39 @@ def _read_health_url(repo: Path) -> str | None:
|
|
|
111
194
|
return None
|
|
112
195
|
|
|
113
196
|
|
|
197
|
+
def resolve_browser_url(repo: Path) -> str | None:
|
|
198
|
+
url = _read_health_url(repo)
|
|
199
|
+
if url:
|
|
200
|
+
return url
|
|
201
|
+
merged = _merge_scratchpad(repo)
|
|
202
|
+
port = merged.get("DEV_SERVER_PORT", "").strip()
|
|
203
|
+
if port:
|
|
204
|
+
return f"http://localhost:{port}/"
|
|
205
|
+
if (repo / "package.json").is_file():
|
|
206
|
+
try:
|
|
207
|
+
pkg = json.loads((repo / "package.json").read_text(encoding="utf-8"))
|
|
208
|
+
if pkg.get("scripts", {}).get("start") or pkg.get("scripts", {}).get("dev"):
|
|
209
|
+
default_port = port or "3000"
|
|
210
|
+
return f"http://localhost:{default_port}/"
|
|
211
|
+
except (json.JSONDecodeError, OSError):
|
|
212
|
+
pass
|
|
213
|
+
return None
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def is_mcp_unavailable() -> bool:
|
|
217
|
+
ci = os.environ.get("CI", "").lower()
|
|
218
|
+
if ci in ("1", "true", "yes"):
|
|
219
|
+
return True
|
|
220
|
+
return os.environ.get("GITHUB_ACTIONS", "").lower() == "true"
|
|
221
|
+
|
|
222
|
+
|
|
114
223
|
def classify_step(step_text: str, repo: Path) -> tuple[str | None, str]:
|
|
115
224
|
if _forbidden(step_text):
|
|
116
225
|
return None, UAT_PROBE_FORBIDDEN
|
|
117
226
|
lower = step_text.lower()
|
|
118
227
|
profile = detect_stack_profile(repo)
|
|
119
228
|
|
|
120
|
-
if
|
|
229
|
+
if _has_judgment_deny(lower):
|
|
121
230
|
return "manual_operator", UAT_PROBE_UNRESOLVED
|
|
122
231
|
|
|
123
232
|
if any(w in lower for w in ("build", "compile", "bundle")):
|
|
@@ -136,21 +245,376 @@ def classify_step(step_text: str, repo: Path) -> tuple[str | None, str]:
|
|
|
136
245
|
return None, UAT_PROBE_UNRESOLVED
|
|
137
246
|
|
|
138
247
|
if any(w in lower for w in ("process", "startup", "server start", "readiness")):
|
|
248
|
+
if _extract_startup_command(step_text, repo) and (
|
|
249
|
+
_read_health_url(repo) or resolve_browser_url(repo)
|
|
250
|
+
):
|
|
251
|
+
return "process_health", ""
|
|
139
252
|
return "process_health", UAT_PROBE_UNRESOLVED
|
|
140
253
|
|
|
254
|
+
if _has_automatable_ui(lower):
|
|
255
|
+
if resolve_browser_url(repo):
|
|
256
|
+
return "browser_smoke", ""
|
|
257
|
+
return None, UAT_PROBE_UNRESOLVED
|
|
258
|
+
|
|
141
259
|
if any(w in lower for w in ("browser", "playwright", "smoke", "ui")):
|
|
142
|
-
if
|
|
260
|
+
if resolve_browser_url(repo):
|
|
143
261
|
return "browser_smoke", ""
|
|
144
262
|
return None, UAT_PROBE_UNRESOLVED
|
|
145
263
|
|
|
146
264
|
if any(w in lower for w in ("cli", "command line", "exit code")):
|
|
265
|
+
if _extract_backtick_command(step_text):
|
|
266
|
+
return "cli_smoke", ""
|
|
147
267
|
return "cli_smoke", UAT_PROBE_UNRESOLVED
|
|
148
268
|
|
|
269
|
+
if any(w in lower for w in ("manual", "operator", "human", "judgment")):
|
|
270
|
+
return "manual_operator", UAT_PROBE_UNRESOLVED
|
|
271
|
+
|
|
149
272
|
if profile is None:
|
|
150
273
|
return None, UAT_STACK_PROFILE_UNKNOWN
|
|
151
274
|
return None, UAT_PROBE_UNRESOLVED
|
|
152
275
|
|
|
153
276
|
|
|
277
|
+
def _extract_backtick_command(text: str) -> str | None:
|
|
278
|
+
m = re.search(r"`([^`]+)`", text)
|
|
279
|
+
return m.group(1).strip() if m else None
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
def _extract_stdout_expectation(text: str) -> str | None:
|
|
283
|
+
m = re.search(r'expect\s+"([^"]+)"', text, re.I)
|
|
284
|
+
if m:
|
|
285
|
+
return m.group(1)
|
|
286
|
+
m = re.search(r'output contains\s+"([^"]+)"', text, re.I)
|
|
287
|
+
return m.group(1) if m else None
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
def _extract_startup_command(step_text: str, repo: Path) -> str | None:
|
|
291
|
+
merged = _merge_scratchpad(repo)
|
|
292
|
+
override = merged.get("DEV_SERVER_COMMAND", "").strip()
|
|
293
|
+
if override:
|
|
294
|
+
return override
|
|
295
|
+
cmd = _extract_backtick_command(step_text)
|
|
296
|
+
if cmd:
|
|
297
|
+
return cmd
|
|
298
|
+
m = re.search(r'"([^"]+(?:start|dev|serve)[^"]*)"', step_text, re.I)
|
|
299
|
+
if m:
|
|
300
|
+
return m.group(1)
|
|
301
|
+
m = re.search(
|
|
302
|
+
r"(npm\s+(?:run\s+)?(?:start|dev)|yarn\s+(?:start|dev)|python\s+-m\s+\w+)",
|
|
303
|
+
step_text,
|
|
304
|
+
re.I,
|
|
305
|
+
)
|
|
306
|
+
if m:
|
|
307
|
+
return m.group(1)
|
|
308
|
+
profile = detect_stack_profile(repo)
|
|
309
|
+
if profile == "node" and (repo / "package.json").is_file():
|
|
310
|
+
try:
|
|
311
|
+
pkg = json.loads((repo / "package.json").read_text(encoding="utf-8"))
|
|
312
|
+
scripts = pkg.get("scripts", {})
|
|
313
|
+
if "dev" in scripts:
|
|
314
|
+
return "npm run dev"
|
|
315
|
+
if "start" in scripts:
|
|
316
|
+
return "npm run start"
|
|
317
|
+
except (json.JSONDecodeError, OSError):
|
|
318
|
+
pass
|
|
319
|
+
return None
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
def _http_get_probe(url: str, timeout: int) -> tuple[bool, str, dict[str, object]]:
|
|
323
|
+
meta: dict[str, object] = {"url": url}
|
|
324
|
+
try:
|
|
325
|
+
req = urllib.request.Request(url, method="GET")
|
|
326
|
+
with urllib.request.urlopen(req, timeout=min(timeout, 30)) as resp:
|
|
327
|
+
meta["status_code"] = resp.status
|
|
328
|
+
if 200 <= resp.status < 400:
|
|
329
|
+
return True, UAT_PROBE_PASS, meta
|
|
330
|
+
return False, UAT_BROWSER_PROBE_FAILED, meta
|
|
331
|
+
except urllib.error.URLError as exc:
|
|
332
|
+
meta["error"] = type(exc).__name__
|
|
333
|
+
return False, UAT_BROWSER_PROBE_FAILED, meta
|
|
334
|
+
except TimeoutError:
|
|
335
|
+
return False, UAT_BROWSER_PROBE_TIMEOUT, meta
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
def _playwright_probe(url: str, timeout: int) -> tuple[bool, str, dict[str, object]]:
|
|
339
|
+
meta: dict[str, object] = {"url": url}
|
|
340
|
+
script = (
|
|
341
|
+
"const { chromium } = require('playwright');"
|
|
342
|
+
"(async () => {"
|
|
343
|
+
f" const b = await chromium.launch(); const p = await b.newPage();"
|
|
344
|
+
f" await p.goto({json.dumps(url)}, {{ timeout: {min(timeout, 30) * 1000} }});"
|
|
345
|
+
" await b.close(); process.exit(0);"
|
|
346
|
+
"})().catch(() => process.exit(1));"
|
|
347
|
+
)
|
|
348
|
+
try:
|
|
349
|
+
proc = subprocess.run(
|
|
350
|
+
["node", "-e", script],
|
|
351
|
+
capture_output=True,
|
|
352
|
+
text=True,
|
|
353
|
+
timeout=min(timeout, 60),
|
|
354
|
+
)
|
|
355
|
+
meta["exit_code"] = proc.returncode
|
|
356
|
+
if proc.returncode == 0:
|
|
357
|
+
return True, UAT_PROBE_PASS, meta
|
|
358
|
+
return False, UAT_BROWSER_PROBE_FAILED, meta
|
|
359
|
+
except subprocess.TimeoutExpired:
|
|
360
|
+
return False, UAT_BROWSER_PROBE_TIMEOUT, meta
|
|
361
|
+
except OSError as exc:
|
|
362
|
+
meta["error"] = str(exc)
|
|
363
|
+
return False, UAT_BROWSER_UNAVAILABLE, meta
|
|
364
|
+
|
|
365
|
+
|
|
366
|
+
def _run_fallback_chain(
|
|
367
|
+
url: str,
|
|
368
|
+
repo: Path,
|
|
369
|
+
timeout: int,
|
|
370
|
+
*,
|
|
371
|
+
dom_interaction: bool = False,
|
|
372
|
+
) -> tuple[bool, str, dict[str, object], str]:
|
|
373
|
+
mode = read_browser_probe_mode(repo)
|
|
374
|
+
chain = read_fallback_chain(repo)
|
|
375
|
+
details: dict[str, object] = {"url": url, "execution_tier": "stdlib"}
|
|
376
|
+
|
|
377
|
+
if mode == "http_fallback":
|
|
378
|
+
ok, code, meta = _http_get_probe(url, timeout)
|
|
379
|
+
details.update(meta)
|
|
380
|
+
return ok, code, details, "http_fallback"
|
|
381
|
+
|
|
382
|
+
if mode == "playwright_fallback":
|
|
383
|
+
ok, code, meta = _playwright_probe(url, timeout)
|
|
384
|
+
details.update(meta)
|
|
385
|
+
if ok:
|
|
386
|
+
return ok, code, details, "playwright_fallback"
|
|
387
|
+
if chain:
|
|
388
|
+
ok, code, meta = _http_get_probe(url, timeout)
|
|
389
|
+
details.update(meta)
|
|
390
|
+
details["fallback_from"] = "playwright_fallback"
|
|
391
|
+
return ok, code, details, "http_fallback"
|
|
392
|
+
return ok, UAT_BROWSER_UNAVAILABLE, details, "playwright_fallback"
|
|
393
|
+
|
|
394
|
+
if dom_interaction and chain:
|
|
395
|
+
ok, code, meta = _playwright_probe(url, timeout)
|
|
396
|
+
if ok:
|
|
397
|
+
details.update(meta)
|
|
398
|
+
return ok, code, details, "playwright_fallback"
|
|
399
|
+
|
|
400
|
+
ok, code, meta = _http_get_probe(url, timeout)
|
|
401
|
+
details.update(meta)
|
|
402
|
+
if ok:
|
|
403
|
+
return ok, code, details, "http_fallback"
|
|
404
|
+
if chain and dom_interaction:
|
|
405
|
+
ok, code, meta = _playwright_probe(url, timeout)
|
|
406
|
+
details.update(meta)
|
|
407
|
+
details["fallback_from"] = "http_fallback"
|
|
408
|
+
return ok, code, details, "playwright_fallback"
|
|
409
|
+
return ok, code, details, "http_fallback"
|
|
410
|
+
|
|
411
|
+
|
|
412
|
+
def execute_browser_smoke(
|
|
413
|
+
step_text: str,
|
|
414
|
+
repo: Path,
|
|
415
|
+
*,
|
|
416
|
+
timeout: int = DEFAULT_PROBE_TIMEOUT,
|
|
417
|
+
) -> dict[str, object]:
|
|
418
|
+
mode = read_browser_probe_mode(repo)
|
|
419
|
+
url = resolve_browser_url(repo)
|
|
420
|
+
dom_interaction = _has_automatable_ui(step_text.lower())
|
|
421
|
+
result: dict[str, object] = {
|
|
422
|
+
"probe_kind": "browser_smoke",
|
|
423
|
+
"step": step_text[:200],
|
|
424
|
+
"stack_profile": detect_stack_profile(repo) or "unknown",
|
|
425
|
+
"probe_mode": mode,
|
|
426
|
+
"passed": False,
|
|
427
|
+
"reason_code": UAT_PROBE_UNRESOLVED,
|
|
428
|
+
"target_url": url,
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
if not url:
|
|
432
|
+
result["reason_code"] = UAT_PROBE_UNRESOLVED
|
|
433
|
+
return result
|
|
434
|
+
|
|
435
|
+
if mode == "cursor":
|
|
436
|
+
if is_mcp_unavailable():
|
|
437
|
+
result["reason_code"] = UAT_BROWSER_UNAVAILABLE
|
|
438
|
+
result["execution_tier"] = "stdlib"
|
|
439
|
+
if read_fallback_chain(repo):
|
|
440
|
+
ok, code, meta, fb_mode = _run_fallback_chain(
|
|
441
|
+
url, repo, timeout, dom_interaction=dom_interaction
|
|
442
|
+
)
|
|
443
|
+
result.update(meta)
|
|
444
|
+
result["probe_mode"] = fb_mode
|
|
445
|
+
result["passed"] = ok
|
|
446
|
+
result["reason_code"] = code if ok else UAT_BROWSER_PROBE_FAILED
|
|
447
|
+
if not ok and code == UAT_BROWSER_UNAVAILABLE:
|
|
448
|
+
result["reason_code"] = UAT_BROWSER_UNAVAILABLE
|
|
449
|
+
return result
|
|
450
|
+
result["execution_tier"] = "agent"
|
|
451
|
+
result["reason_code"] = UAT_PROBE_UNRESOLVED
|
|
452
|
+
result["agent_plan"] = {
|
|
453
|
+
"sequence": "browser_navigate → interact → screenshot → console/network → evidence",
|
|
454
|
+
"evidence_dir": "sprints/Sxxxx/evidence/browser/",
|
|
455
|
+
}
|
|
456
|
+
return result
|
|
457
|
+
|
|
458
|
+
result["execution_tier"] = "stdlib"
|
|
459
|
+
ok, code, meta, fb_mode = _run_fallback_chain(
|
|
460
|
+
url, repo, timeout, dom_interaction=dom_interaction
|
|
461
|
+
)
|
|
462
|
+
result.update(meta)
|
|
463
|
+
result["probe_mode"] = fb_mode
|
|
464
|
+
result["passed"] = ok
|
|
465
|
+
result["reason_code"] = code
|
|
466
|
+
return result
|
|
467
|
+
|
|
468
|
+
|
|
469
|
+
def execute_process_health(
|
|
470
|
+
step_text: str,
|
|
471
|
+
repo: Path,
|
|
472
|
+
*,
|
|
473
|
+
timeout: int = DEFAULT_PROBE_TIMEOUT,
|
|
474
|
+
poll_cap: int | None = None,
|
|
475
|
+
poll_interval: int | None = None,
|
|
476
|
+
) -> dict[str, object]:
|
|
477
|
+
cap, interval = read_poll_settings(repo)
|
|
478
|
+
if poll_cap is not None:
|
|
479
|
+
cap = poll_cap
|
|
480
|
+
if poll_interval is not None:
|
|
481
|
+
interval = poll_interval
|
|
482
|
+
|
|
483
|
+
startup = _extract_startup_command(step_text, repo)
|
|
484
|
+
health_url = _read_health_url(repo) or resolve_browser_url(repo)
|
|
485
|
+
result: dict[str, object] = {
|
|
486
|
+
"probe_kind": "process_health",
|
|
487
|
+
"step": step_text[:200],
|
|
488
|
+
"stack_profile": detect_stack_profile(repo) or "unknown",
|
|
489
|
+
"execution_tier": "stdlib",
|
|
490
|
+
"passed": False,
|
|
491
|
+
"reason_code": UAT_PROBE_UNRESOLVED,
|
|
492
|
+
}
|
|
493
|
+
|
|
494
|
+
if not startup or not health_url:
|
|
495
|
+
return result
|
|
496
|
+
|
|
497
|
+
result["startup_command"] = startup
|
|
498
|
+
result["health_url"] = health_url
|
|
499
|
+
proc: subprocess.Popen[str] | None = None
|
|
500
|
+
try:
|
|
501
|
+
proc = subprocess.Popen(
|
|
502
|
+
startup,
|
|
503
|
+
shell=True,
|
|
504
|
+
cwd=str(repo),
|
|
505
|
+
stdout=subprocess.DEVNULL,
|
|
506
|
+
stderr=subprocess.DEVNULL,
|
|
507
|
+
)
|
|
508
|
+
deadline = time.monotonic() + cap
|
|
509
|
+
while time.monotonic() < deadline:
|
|
510
|
+
ok, code, meta = _http_get_probe(health_url, min(timeout, interval + 2))
|
|
511
|
+
if ok:
|
|
512
|
+
result.update(meta)
|
|
513
|
+
result["passed"] = True
|
|
514
|
+
result["reason_code"] = UAT_PROBE_PASS
|
|
515
|
+
return result
|
|
516
|
+
time.sleep(interval)
|
|
517
|
+
result["reason_code"] = UAT_PROBE_TIMEOUT
|
|
518
|
+
except OSError as exc:
|
|
519
|
+
result["reason_code"] = UAT_PROBE_FAILED
|
|
520
|
+
result["error"] = str(exc)
|
|
521
|
+
finally:
|
|
522
|
+
if proc is not None:
|
|
523
|
+
proc.terminate()
|
|
524
|
+
try:
|
|
525
|
+
proc.wait(timeout=5)
|
|
526
|
+
except subprocess.TimeoutExpired:
|
|
527
|
+
proc.kill()
|
|
528
|
+
return result
|
|
529
|
+
|
|
530
|
+
|
|
531
|
+
def execute_cli_smoke(
|
|
532
|
+
step_text: str,
|
|
533
|
+
repo: Path,
|
|
534
|
+
*,
|
|
535
|
+
timeout: int = DEFAULT_PROBE_TIMEOUT,
|
|
536
|
+
) -> dict[str, object]:
|
|
537
|
+
cmd = _extract_backtick_command(step_text)
|
|
538
|
+
expect_out = _extract_stdout_expectation(step_text)
|
|
539
|
+
result: dict[str, object] = {
|
|
540
|
+
"probe_kind": "cli_smoke",
|
|
541
|
+
"step": step_text[:200],
|
|
542
|
+
"stack_profile": detect_stack_profile(repo) or "unknown",
|
|
543
|
+
"execution_tier": "stdlib",
|
|
544
|
+
"passed": False,
|
|
545
|
+
"reason_code": UAT_PROBE_UNRESOLVED,
|
|
546
|
+
}
|
|
547
|
+
if not cmd:
|
|
548
|
+
return result
|
|
549
|
+
result["command"] = cmd
|
|
550
|
+
try:
|
|
551
|
+
proc = subprocess.run(
|
|
552
|
+
cmd,
|
|
553
|
+
shell=True,
|
|
554
|
+
cwd=str(repo),
|
|
555
|
+
timeout=timeout,
|
|
556
|
+
capture_output=True,
|
|
557
|
+
text=True,
|
|
558
|
+
)
|
|
559
|
+
result["exit_code"] = proc.returncode
|
|
560
|
+
if proc.returncode != 0:
|
|
561
|
+
result["reason_code"] = UAT_PROBE_FAILED
|
|
562
|
+
return result
|
|
563
|
+
if expect_out and expect_out not in proc.stdout:
|
|
564
|
+
result["reason_code"] = UAT_PROBE_FAILED
|
|
565
|
+
return result
|
|
566
|
+
result["reason_code"] = UAT_PROBE_PASS
|
|
567
|
+
result["passed"] = True
|
|
568
|
+
except subprocess.TimeoutExpired:
|
|
569
|
+
result["reason_code"] = UAT_PROBE_TIMEOUT
|
|
570
|
+
except OSError as exc:
|
|
571
|
+
result["reason_code"] = UAT_PROBE_FAILED
|
|
572
|
+
result["error"] = str(exc)
|
|
573
|
+
return result
|
|
574
|
+
|
|
575
|
+
|
|
576
|
+
def validate_browser_evidence(result: dict[str, object], mode: str) -> dict[str, object]:
|
|
577
|
+
out = dict(result)
|
|
578
|
+
if not out.get("passed"):
|
|
579
|
+
return out
|
|
580
|
+
if mode != "cursor":
|
|
581
|
+
return out
|
|
582
|
+
refs = out.get("browser_evidence_refs")
|
|
583
|
+
if not isinstance(refs, dict):
|
|
584
|
+
out["passed"] = False
|
|
585
|
+
out["reason_code"] = UAT_BROWSER_PROBE_FAILED
|
|
586
|
+
return out
|
|
587
|
+
nav = str(refs.get("navigation_url") or "").strip()
|
|
588
|
+
screenshots = refs.get("screenshots")
|
|
589
|
+
if not isinstance(screenshots, list):
|
|
590
|
+
screenshots = []
|
|
591
|
+
console = refs.get("console_summary")
|
|
592
|
+
network = refs.get("network_summary")
|
|
593
|
+
console_path = ""
|
|
594
|
+
network_path = ""
|
|
595
|
+
if isinstance(console, dict):
|
|
596
|
+
console_path = str(console.get("summary_path") or "")
|
|
597
|
+
if isinstance(network, dict):
|
|
598
|
+
network_path = str(network.get("summary_path") or "")
|
|
599
|
+
has_evidence = bool(nav) and (
|
|
600
|
+
bool(screenshots) or bool(console_path) or bool(network_path)
|
|
601
|
+
)
|
|
602
|
+
if len(screenshots) > MAX_SCREENSHOTS:
|
|
603
|
+
has_evidence = False
|
|
604
|
+
if not has_evidence:
|
|
605
|
+
out["passed"] = False
|
|
606
|
+
out["reason_code"] = UAT_BROWSER_PROBE_FAILED
|
|
607
|
+
return out
|
|
608
|
+
|
|
609
|
+
|
|
610
|
+
def merge_result_fragment(fragment: dict[str, object], repo: Path) -> dict[str, object]:
|
|
611
|
+
mode = read_browser_probe_mode(repo)
|
|
612
|
+
if "probe_mode" not in fragment:
|
|
613
|
+
fragment = dict(fragment)
|
|
614
|
+
fragment["probe_mode"] = mode
|
|
615
|
+
return validate_browser_evidence(fragment, mode)
|
|
616
|
+
|
|
617
|
+
|
|
154
618
|
def execute_probe(
|
|
155
619
|
kind: str,
|
|
156
620
|
step_text: str,
|
|
@@ -176,6 +640,7 @@ def execute_probe(
|
|
|
176
640
|
if not cmd:
|
|
177
641
|
result["reason_code"] = UAT_PROBE_UNRESOLVED
|
|
178
642
|
return result
|
|
643
|
+
result["execution_tier"] = "stdlib"
|
|
179
644
|
return _run_subprocess(cmd, repo, timeout, result)
|
|
180
645
|
|
|
181
646
|
if kind == "test":
|
|
@@ -190,6 +655,7 @@ def execute_probe(
|
|
|
190
655
|
else:
|
|
191
656
|
result["reason_code"] = UAT_PROBE_UNRESOLVED
|
|
192
657
|
return result
|
|
658
|
+
result["execution_tier"] = "stdlib"
|
|
193
659
|
return _run_subprocess(cmd, repo, timeout, result)
|
|
194
660
|
|
|
195
661
|
if kind == "api_health":
|
|
@@ -197,24 +663,21 @@ def execute_probe(
|
|
|
197
663
|
if not url:
|
|
198
664
|
result["reason_code"] = UAT_PROBE_UNRESOLVED
|
|
199
665
|
return result
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
result["passed"] = 200 <= resp.status < 400
|
|
206
|
-
if not result["passed"]:
|
|
207
|
-
result["reason_code"] = UAT_PROBE_FAILED
|
|
208
|
-
except urllib.error.URLError as exc:
|
|
209
|
-
result["reason_code"] = UAT_PROBE_FAILED
|
|
210
|
-
result["error"] = type(exc).__name__
|
|
211
|
-
except TimeoutError:
|
|
212
|
-
result["reason_code"] = UAT_PROBE_TIMEOUT
|
|
666
|
+
result["execution_tier"] = "stdlib"
|
|
667
|
+
ok, code, meta = _http_get_probe(url, timeout)
|
|
668
|
+
result.update(meta)
|
|
669
|
+
result["passed"] = ok
|
|
670
|
+
result["reason_code"] = code if ok else UAT_PROBE_FAILED
|
|
213
671
|
return result
|
|
214
672
|
|
|
215
|
-
if kind
|
|
216
|
-
|
|
217
|
-
|
|
673
|
+
if kind == "process_health":
|
|
674
|
+
return execute_process_health(step_text, repo, timeout=timeout)
|
|
675
|
+
|
|
676
|
+
if kind == "browser_smoke":
|
|
677
|
+
return execute_browser_smoke(step_text, repo, timeout=timeout)
|
|
678
|
+
|
|
679
|
+
if kind == "cli_smoke":
|
|
680
|
+
return execute_cli_smoke(step_text, repo, timeout=timeout)
|
|
218
681
|
|
|
219
682
|
result["reason_code"] = UAT_PROBE_UNRESOLVED
|
|
220
683
|
return result
|
|
@@ -260,6 +723,12 @@ def resolve_and_probe(step_text: str, repo: Path) -> dict[str, object]:
|
|
|
260
723
|
}
|
|
261
724
|
if pre_reason == UAT_PROBE_FORBIDDEN:
|
|
262
725
|
return {"probe_kind": kind, "reason_code": UAT_PROBE_FORBIDDEN, "passed": False}
|
|
726
|
+
if pre_reason == UAT_PROBE_UNRESOLVED and kind in ("manual_operator",):
|
|
727
|
+
return {
|
|
728
|
+
"probe_kind": kind,
|
|
729
|
+
"reason_code": UAT_PROBE_UNRESOLVED,
|
|
730
|
+
"passed": False,
|
|
731
|
+
}
|
|
263
732
|
return execute_probe(kind, step_text, Path(repo))
|
|
264
733
|
|
|
265
734
|
|
|
@@ -269,25 +738,93 @@ def probe_steps(steps: list[str], repo: Path) -> list[dict[str, object]]:
|
|
|
269
738
|
|
|
270
739
|
def self_test() -> None:
|
|
271
740
|
repo = Path(__file__).resolve().parents[1]
|
|
272
|
-
assert UAT_PROBE_PASS in PROBE_KINDS or UAT_PROBE_PASS == "UAT_PROBE_PASS"
|
|
273
741
|
assert "build" in PROBE_KINDS
|
|
742
|
+
|
|
274
743
|
r = classify_step("run unit tests", repo)
|
|
275
744
|
assert r[0] == "test" or r[1] in (UAT_PROBE_UNRESOLVED, "")
|
|
745
|
+
|
|
276
746
|
r2 = classify_step("read secrets from .env file", repo)
|
|
277
747
|
assert r2[1] == UAT_PROBE_FORBIDDEN
|
|
278
|
-
|
|
279
|
-
|
|
748
|
+
|
|
749
|
+
r3 = classify_step("enter password in login form", repo)
|
|
750
|
+
assert r3[1] == UAT_PROBE_FORBIDDEN
|
|
751
|
+
|
|
752
|
+
r4 = classify_step("operator visually confirms button click", repo)
|
|
753
|
+
assert r4[0] == "manual_operator"
|
|
754
|
+
assert r4[1] == UAT_PROBE_UNRESOLVED
|
|
755
|
+
|
|
756
|
+
r5 = classify_step("click submit button on home page", repo)
|
|
757
|
+
assert r5[0] == "browser_smoke" or r5[1] == UAT_PROBE_UNRESOLVED
|
|
758
|
+
|
|
759
|
+
r6 = classify_step("operator verifies compliance manually", repo)
|
|
760
|
+
assert r6[0] == "manual_operator"
|
|
761
|
+
|
|
762
|
+
mode = read_browser_probe_mode(repo)
|
|
763
|
+
assert mode in BROWSER_PROBE_MODES
|
|
764
|
+
|
|
765
|
+
bs = execute_browser_smoke("browser smoke test ui", repo)
|
|
766
|
+
assert bs.get("probe_mode") == "cursor"
|
|
767
|
+
assert bs["passed"] is False
|
|
768
|
+
if bs.get("target_url"):
|
|
769
|
+
assert bs.get("execution_tier") == "agent"
|
|
770
|
+
assert bs["reason_code"] == UAT_PROBE_UNRESOLVED
|
|
771
|
+
else:
|
|
772
|
+
assert bs["reason_code"] == UAT_PROBE_UNRESOLVED
|
|
773
|
+
|
|
774
|
+
bad_pass = validate_browser_evidence(
|
|
775
|
+
{"passed": True, "reason_code": UAT_PROBE_PASS},
|
|
776
|
+
"cursor",
|
|
777
|
+
)
|
|
778
|
+
assert bad_pass["passed"] is False
|
|
779
|
+
assert bad_pass["reason_code"] == UAT_BROWSER_PROBE_FAILED
|
|
780
|
+
|
|
781
|
+
good_pass = validate_browser_evidence(
|
|
782
|
+
{
|
|
783
|
+
"passed": True,
|
|
784
|
+
"reason_code": UAT_PROBE_PASS,
|
|
785
|
+
"browser_evidence_refs": {
|
|
786
|
+
"navigation_url": "http://localhost:3000/",
|
|
787
|
+
"screenshots": ["sprints/S0082/evidence/browser/pr-01.png"],
|
|
788
|
+
},
|
|
789
|
+
},
|
|
790
|
+
"cursor",
|
|
791
|
+
)
|
|
792
|
+
assert good_pass["passed"] is True
|
|
793
|
+
|
|
794
|
+
cli = execute_cli_smoke('run CLI `python -c "import sys; sys.exit(0)"` exit code 0', repo)
|
|
795
|
+
assert cli["passed"] is True
|
|
796
|
+
assert cli["reason_code"] == UAT_PROBE_PASS
|
|
797
|
+
|
|
798
|
+
ph = execute_process_health(
|
|
799
|
+
"wait for server startup readiness",
|
|
800
|
+
repo,
|
|
801
|
+
poll_cap=1,
|
|
802
|
+
poll_interval=1,
|
|
803
|
+
)
|
|
804
|
+
assert ph["reason_code"] in (UAT_PROBE_UNRESOLVED, UAT_PROBE_TIMEOUT, UAT_PROBE_FAILED)
|
|
805
|
+
|
|
806
|
+
merged = merge_result_fragment(
|
|
807
|
+
{"passed": True, "probe_kind": "browser_smoke"},
|
|
808
|
+
repo,
|
|
809
|
+
)
|
|
810
|
+
assert merged["passed"] is False
|
|
811
|
+
|
|
280
812
|
assert detect_stack_profile(repo) in ("python", "node", None, "generated")
|
|
281
813
|
|
|
282
814
|
|
|
283
815
|
def main() -> int:
|
|
284
816
|
import argparse
|
|
285
817
|
|
|
286
|
-
parser = argparse.ArgumentParser(description="UAT probe resolver (US-0092).")
|
|
818
|
+
parser = argparse.ArgumentParser(description="UAT probe resolver (US-0092 / US-0093).")
|
|
287
819
|
parser.add_argument("--repo", default=".")
|
|
288
820
|
parser.add_argument("--step", action="append", default=[], help="Acceptance step text.")
|
|
289
821
|
parser.add_argument("--self-test", action="store_true")
|
|
290
822
|
parser.add_argument("--report", action="store_true", help="JSON probe results to stdout.")
|
|
823
|
+
parser.add_argument(
|
|
824
|
+
"--merge-result",
|
|
825
|
+
metavar="FRAGMENT.json",
|
|
826
|
+
help="Validate browser evidence fragment JSON (US-0093).",
|
|
827
|
+
)
|
|
291
828
|
args = parser.parse_args()
|
|
292
829
|
repo = Path(args.repo).resolve()
|
|
293
830
|
|
|
@@ -300,6 +837,20 @@ def main() -> int:
|
|
|
300
837
|
print("[UAT_PROBE_LIB_SELF_TEST_OK]")
|
|
301
838
|
return 0
|
|
302
839
|
|
|
840
|
+
if args.merge_result:
|
|
841
|
+
frag_path = Path(args.merge_result)
|
|
842
|
+
if not frag_path.is_file():
|
|
843
|
+
print(f"merge-result: file not found: {frag_path}", file=sys.stderr)
|
|
844
|
+
return 2
|
|
845
|
+
try:
|
|
846
|
+
fragment = json.loads(frag_path.read_text(encoding="utf-8"))
|
|
847
|
+
except json.JSONDecodeError as exc:
|
|
848
|
+
print(f"merge-result: invalid JSON: {exc}", file=sys.stderr)
|
|
849
|
+
return 2
|
|
850
|
+
validated = merge_result_fragment(fragment, repo)
|
|
851
|
+
print(json.dumps(validated, sort_keys=True, separators=(",", ":")))
|
|
852
|
+
return 0 if validated.get("passed") else 1
|
|
853
|
+
|
|
303
854
|
if args.report or args.step:
|
|
304
855
|
results = probe_steps(args.step or ["run tests"], repo)
|
|
305
856
|
print(json.dumps(results, sort_keys=True, separators=(",", ":")))
|