its-magic 0.1.2-48 → 0.1.2-55
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +61 -18
- package/package.json +1 -1
- package/scripts/check_intake_template_parity.py +40 -0
- package/template/.cursor/commands/auto.md +82 -6
- package/template/.cursor/commands/execute.md +49 -0
- package/template/.cursor/commands/qa.md +29 -0
- package/template/.cursor/commands/verify-work.md +28 -1
- package/template/.cursor/scratchpad.local.example.md +33 -4
- package/template/.cursor/scratchpad.md +20 -0
- package/template/README.md +67 -18
- package/template/docs/engineering/auto-orchestration-reference.md +115 -15
- package/template/docs/engineering/runbook.md +71 -5
- package/template/scripts/check_intake_template_parity.py +40 -0
- package/template/scripts/uat_probe_lib.py +576 -25
|
@@ -1,17 +1,19 @@
|
|
|
1
1
|
#!/usr/bin/env python3
|
|
2
2
|
"""
|
|
3
|
-
UAT probe resolver (US-0092 / DEC-0078).
|
|
3
|
+
UAT probe resolver (US-0092 / DEC-0078; US-0093 / DEC-0079).
|
|
4
4
|
|
|
5
5
|
Shared by /verify-work and /qa for self-verify acceptance steps.
|
|
6
|
-
Fail-closed — no silent PASS.
|
|
6
|
+
Fail-closed — no silent PASS. Lib never invokes browser MCP (BUG-0006).
|
|
7
7
|
"""
|
|
8
8
|
|
|
9
9
|
from __future__ import annotations
|
|
10
10
|
|
|
11
11
|
import json
|
|
12
|
+
import os
|
|
12
13
|
import re
|
|
13
14
|
import subprocess
|
|
14
15
|
import sys
|
|
16
|
+
import time
|
|
15
17
|
import urllib.error
|
|
16
18
|
import urllib.request
|
|
17
19
|
from pathlib import Path
|
|
@@ -22,6 +24,9 @@ UAT_PROBE_TIMEOUT = "UAT_PROBE_TIMEOUT"
|
|
|
22
24
|
UAT_PROBE_FAILED = "UAT_PROBE_FAILED"
|
|
23
25
|
UAT_PROBE_FORBIDDEN = "UAT_PROBE_FORBIDDEN"
|
|
24
26
|
UAT_PROBE_PASS = "UAT_PROBE_PASS"
|
|
27
|
+
UAT_BROWSER_UNAVAILABLE = "UAT_BROWSER_UNAVAILABLE"
|
|
28
|
+
UAT_BROWSER_PROBE_FAILED = "UAT_BROWSER_PROBE_FAILED"
|
|
29
|
+
UAT_BROWSER_PROBE_TIMEOUT = "UAT_BROWSER_PROBE_TIMEOUT"
|
|
25
30
|
|
|
26
31
|
PROBE_KINDS = (
|
|
27
32
|
"build",
|
|
@@ -34,8 +39,43 @@ PROBE_KINDS = (
|
|
|
34
39
|
)
|
|
35
40
|
|
|
36
41
|
FORBIDDEN_PATH_TOKENS = (".env", "intake_evidence", "handoffs/intake_evidence")
|
|
42
|
+
SECRET_FORBIDDEN_TOKENS = ("password", "credential", "api key")
|
|
43
|
+
|
|
44
|
+
JUDGMENT_DENY_TOKENS = (
|
|
45
|
+
"visually",
|
|
46
|
+
"aesthetically",
|
|
47
|
+
"operator confirms",
|
|
48
|
+
"subjective",
|
|
49
|
+
"human judgment",
|
|
50
|
+
"eyeball",
|
|
51
|
+
"manually verify appearance",
|
|
52
|
+
"approve layout",
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
AUTOMATABLE_UI_TOKENS = (
|
|
56
|
+
"click",
|
|
57
|
+
"fill",
|
|
58
|
+
"navigate",
|
|
59
|
+
"smoke",
|
|
60
|
+
"form",
|
|
61
|
+
"submit",
|
|
62
|
+
"button",
|
|
63
|
+
"page load",
|
|
64
|
+
"scroll",
|
|
65
|
+
"type into",
|
|
66
|
+
"select",
|
|
67
|
+
"checkbox",
|
|
68
|
+
"dropdown",
|
|
69
|
+
"ui",
|
|
70
|
+
"browser",
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
BROWSER_PROBE_MODES = ("cursor", "http_fallback", "playwright_fallback")
|
|
37
74
|
|
|
38
75
|
DEFAULT_PROBE_TIMEOUT = 120
|
|
76
|
+
DEFAULT_POLL_SECONDS = 60
|
|
77
|
+
DEFAULT_POLL_INTERVAL = 2
|
|
78
|
+
MAX_SCREENSHOTS = 5
|
|
39
79
|
|
|
40
80
|
|
|
41
81
|
def _merge_scratchpad(repo: Path) -> dict[str, str]:
|
|
@@ -54,6 +94,39 @@ def _merge_scratchpad(repo: Path) -> dict[str, str]:
|
|
|
54
94
|
return values
|
|
55
95
|
|
|
56
96
|
|
|
97
|
+
def _read_int(merged: dict[str, str], key: str, default: int) -> int:
|
|
98
|
+
raw = merged.get(key, "")
|
|
99
|
+
if not raw:
|
|
100
|
+
return default
|
|
101
|
+
try:
|
|
102
|
+
return max(1, int(raw))
|
|
103
|
+
except ValueError:
|
|
104
|
+
return default
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def read_browser_probe_mode(repo: Path) -> str:
|
|
108
|
+
merged = _merge_scratchpad(repo)
|
|
109
|
+
mode = merged.get("UAT_BROWSER_PROBE_MODE", "cursor").strip()
|
|
110
|
+
return mode if mode in BROWSER_PROBE_MODES else "cursor"
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def read_fallback_chain(repo: Path) -> bool:
|
|
114
|
+
merged = _merge_scratchpad(repo)
|
|
115
|
+
raw = merged.get("UAT_BROWSER_FALLBACK_CHAIN", "1").strip()
|
|
116
|
+
if raw in ("0", "1"):
|
|
117
|
+
return raw == "1"
|
|
118
|
+
return True
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def read_poll_settings(repo: Path) -> tuple[int, int]:
|
|
122
|
+
merged = _merge_scratchpad(repo)
|
|
123
|
+
cap = _read_int(merged, "UAT_PROCESS_HEALTH_POLL_SECONDS", DEFAULT_POLL_SECONDS)
|
|
124
|
+
interval = _read_int(
|
|
125
|
+
merged, "UAT_PROCESS_HEALTH_POLL_INTERVAL_SECONDS", DEFAULT_POLL_INTERVAL
|
|
126
|
+
)
|
|
127
|
+
return cap, interval
|
|
128
|
+
|
|
129
|
+
|
|
57
130
|
def detect_stack_profile(repo: Path) -> str | None:
|
|
58
131
|
if (repo / "package.json").is_file():
|
|
59
132
|
return "node"
|
|
@@ -73,7 +146,17 @@ def detect_stack_profile(repo: Path) -> str | None:
|
|
|
73
146
|
|
|
74
147
|
def _forbidden(step_text: str) -> bool:
|
|
75
148
|
lower = step_text.lower()
|
|
76
|
-
|
|
149
|
+
if any(tok in lower for tok in FORBIDDEN_PATH_TOKENS):
|
|
150
|
+
return True
|
|
151
|
+
return any(tok in lower for tok in SECRET_FORBIDDEN_TOKENS)
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def _has_judgment_deny(lower: str) -> bool:
|
|
155
|
+
return any(tok in lower for tok in JUDGMENT_DENY_TOKENS)
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def _has_automatable_ui(lower: str) -> bool:
|
|
159
|
+
return any(tok in lower for tok in AUTOMATABLE_UI_TOKENS)
|
|
77
160
|
|
|
78
161
|
|
|
79
162
|
def _read_test_command(repo: Path) -> str | None:
|
|
@@ -111,13 +194,39 @@ def _read_health_url(repo: Path) -> str | None:
|
|
|
111
194
|
return None
|
|
112
195
|
|
|
113
196
|
|
|
197
|
+
def resolve_browser_url(repo: Path) -> str | None:
|
|
198
|
+
url = _read_health_url(repo)
|
|
199
|
+
if url:
|
|
200
|
+
return url
|
|
201
|
+
merged = _merge_scratchpad(repo)
|
|
202
|
+
port = merged.get("DEV_SERVER_PORT", "").strip()
|
|
203
|
+
if port:
|
|
204
|
+
return f"http://localhost:{port}/"
|
|
205
|
+
if (repo / "package.json").is_file():
|
|
206
|
+
try:
|
|
207
|
+
pkg = json.loads((repo / "package.json").read_text(encoding="utf-8"))
|
|
208
|
+
if pkg.get("scripts", {}).get("start") or pkg.get("scripts", {}).get("dev"):
|
|
209
|
+
default_port = port or "3000"
|
|
210
|
+
return f"http://localhost:{default_port}/"
|
|
211
|
+
except (json.JSONDecodeError, OSError):
|
|
212
|
+
pass
|
|
213
|
+
return None
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def is_mcp_unavailable() -> bool:
|
|
217
|
+
ci = os.environ.get("CI", "").lower()
|
|
218
|
+
if ci in ("1", "true", "yes"):
|
|
219
|
+
return True
|
|
220
|
+
return os.environ.get("GITHUB_ACTIONS", "").lower() == "true"
|
|
221
|
+
|
|
222
|
+
|
|
114
223
|
def classify_step(step_text: str, repo: Path) -> tuple[str | None, str]:
|
|
115
224
|
if _forbidden(step_text):
|
|
116
225
|
return None, UAT_PROBE_FORBIDDEN
|
|
117
226
|
lower = step_text.lower()
|
|
118
227
|
profile = detect_stack_profile(repo)
|
|
119
228
|
|
|
120
|
-
if
|
|
229
|
+
if _has_judgment_deny(lower):
|
|
121
230
|
return "manual_operator", UAT_PROBE_UNRESOLVED
|
|
122
231
|
|
|
123
232
|
if any(w in lower for w in ("build", "compile", "bundle")):
|
|
@@ -136,21 +245,376 @@ def classify_step(step_text: str, repo: Path) -> tuple[str | None, str]:
|
|
|
136
245
|
return None, UAT_PROBE_UNRESOLVED
|
|
137
246
|
|
|
138
247
|
if any(w in lower for w in ("process", "startup", "server start", "readiness")):
|
|
248
|
+
if _extract_startup_command(step_text, repo) and (
|
|
249
|
+
_read_health_url(repo) or resolve_browser_url(repo)
|
|
250
|
+
):
|
|
251
|
+
return "process_health", ""
|
|
139
252
|
return "process_health", UAT_PROBE_UNRESOLVED
|
|
140
253
|
|
|
254
|
+
if _has_automatable_ui(lower):
|
|
255
|
+
if resolve_browser_url(repo):
|
|
256
|
+
return "browser_smoke", ""
|
|
257
|
+
return None, UAT_PROBE_UNRESOLVED
|
|
258
|
+
|
|
141
259
|
if any(w in lower for w in ("browser", "playwright", "smoke", "ui")):
|
|
142
|
-
if
|
|
260
|
+
if resolve_browser_url(repo):
|
|
143
261
|
return "browser_smoke", ""
|
|
144
262
|
return None, UAT_PROBE_UNRESOLVED
|
|
145
263
|
|
|
146
264
|
if any(w in lower for w in ("cli", "command line", "exit code")):
|
|
265
|
+
if _extract_backtick_command(step_text):
|
|
266
|
+
return "cli_smoke", ""
|
|
147
267
|
return "cli_smoke", UAT_PROBE_UNRESOLVED
|
|
148
268
|
|
|
269
|
+
if any(w in lower for w in ("manual", "operator", "human", "judgment")):
|
|
270
|
+
return "manual_operator", UAT_PROBE_UNRESOLVED
|
|
271
|
+
|
|
149
272
|
if profile is None:
|
|
150
273
|
return None, UAT_STACK_PROFILE_UNKNOWN
|
|
151
274
|
return None, UAT_PROBE_UNRESOLVED
|
|
152
275
|
|
|
153
276
|
|
|
277
|
+
def _extract_backtick_command(text: str) -> str | None:
|
|
278
|
+
m = re.search(r"`([^`]+)`", text)
|
|
279
|
+
return m.group(1).strip() if m else None
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
def _extract_stdout_expectation(text: str) -> str | None:
|
|
283
|
+
m = re.search(r'expect\s+"([^"]+)"', text, re.I)
|
|
284
|
+
if m:
|
|
285
|
+
return m.group(1)
|
|
286
|
+
m = re.search(r'output contains\s+"([^"]+)"', text, re.I)
|
|
287
|
+
return m.group(1) if m else None
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
def _extract_startup_command(step_text: str, repo: Path) -> str | None:
|
|
291
|
+
merged = _merge_scratchpad(repo)
|
|
292
|
+
override = merged.get("DEV_SERVER_COMMAND", "").strip()
|
|
293
|
+
if override:
|
|
294
|
+
return override
|
|
295
|
+
cmd = _extract_backtick_command(step_text)
|
|
296
|
+
if cmd:
|
|
297
|
+
return cmd
|
|
298
|
+
m = re.search(r'"([^"]+(?:start|dev|serve)[^"]*)"', step_text, re.I)
|
|
299
|
+
if m:
|
|
300
|
+
return m.group(1)
|
|
301
|
+
m = re.search(
|
|
302
|
+
r"(npm\s+(?:run\s+)?(?:start|dev)|yarn\s+(?:start|dev)|python\s+-m\s+\w+)",
|
|
303
|
+
step_text,
|
|
304
|
+
re.I,
|
|
305
|
+
)
|
|
306
|
+
if m:
|
|
307
|
+
return m.group(1)
|
|
308
|
+
profile = detect_stack_profile(repo)
|
|
309
|
+
if profile == "node" and (repo / "package.json").is_file():
|
|
310
|
+
try:
|
|
311
|
+
pkg = json.loads((repo / "package.json").read_text(encoding="utf-8"))
|
|
312
|
+
scripts = pkg.get("scripts", {})
|
|
313
|
+
if "dev" in scripts:
|
|
314
|
+
return "npm run dev"
|
|
315
|
+
if "start" in scripts:
|
|
316
|
+
return "npm run start"
|
|
317
|
+
except (json.JSONDecodeError, OSError):
|
|
318
|
+
pass
|
|
319
|
+
return None
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
def _http_get_probe(url: str, timeout: int) -> tuple[bool, str, dict[str, object]]:
|
|
323
|
+
meta: dict[str, object] = {"url": url}
|
|
324
|
+
try:
|
|
325
|
+
req = urllib.request.Request(url, method="GET")
|
|
326
|
+
with urllib.request.urlopen(req, timeout=min(timeout, 30)) as resp:
|
|
327
|
+
meta["status_code"] = resp.status
|
|
328
|
+
if 200 <= resp.status < 400:
|
|
329
|
+
return True, UAT_PROBE_PASS, meta
|
|
330
|
+
return False, UAT_BROWSER_PROBE_FAILED, meta
|
|
331
|
+
except urllib.error.URLError as exc:
|
|
332
|
+
meta["error"] = type(exc).__name__
|
|
333
|
+
return False, UAT_BROWSER_PROBE_FAILED, meta
|
|
334
|
+
except TimeoutError:
|
|
335
|
+
return False, UAT_BROWSER_PROBE_TIMEOUT, meta
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
def _playwright_probe(url: str, timeout: int) -> tuple[bool, str, dict[str, object]]:
|
|
339
|
+
meta: dict[str, object] = {"url": url}
|
|
340
|
+
script = (
|
|
341
|
+
"const { chromium } = require('playwright');"
|
|
342
|
+
"(async () => {"
|
|
343
|
+
f" const b = await chromium.launch(); const p = await b.newPage();"
|
|
344
|
+
f" await p.goto({json.dumps(url)}, {{ timeout: {min(timeout, 30) * 1000} }});"
|
|
345
|
+
" await b.close(); process.exit(0);"
|
|
346
|
+
"})().catch(() => process.exit(1));"
|
|
347
|
+
)
|
|
348
|
+
try:
|
|
349
|
+
proc = subprocess.run(
|
|
350
|
+
["node", "-e", script],
|
|
351
|
+
capture_output=True,
|
|
352
|
+
text=True,
|
|
353
|
+
timeout=min(timeout, 60),
|
|
354
|
+
)
|
|
355
|
+
meta["exit_code"] = proc.returncode
|
|
356
|
+
if proc.returncode == 0:
|
|
357
|
+
return True, UAT_PROBE_PASS, meta
|
|
358
|
+
return False, UAT_BROWSER_PROBE_FAILED, meta
|
|
359
|
+
except subprocess.TimeoutExpired:
|
|
360
|
+
return False, UAT_BROWSER_PROBE_TIMEOUT, meta
|
|
361
|
+
except OSError as exc:
|
|
362
|
+
meta["error"] = str(exc)
|
|
363
|
+
return False, UAT_BROWSER_UNAVAILABLE, meta
|
|
364
|
+
|
|
365
|
+
|
|
366
|
+
def _run_fallback_chain(
|
|
367
|
+
url: str,
|
|
368
|
+
repo: Path,
|
|
369
|
+
timeout: int,
|
|
370
|
+
*,
|
|
371
|
+
dom_interaction: bool = False,
|
|
372
|
+
) -> tuple[bool, str, dict[str, object], str]:
|
|
373
|
+
mode = read_browser_probe_mode(repo)
|
|
374
|
+
chain = read_fallback_chain(repo)
|
|
375
|
+
details: dict[str, object] = {"url": url, "execution_tier": "stdlib"}
|
|
376
|
+
|
|
377
|
+
if mode == "http_fallback":
|
|
378
|
+
ok, code, meta = _http_get_probe(url, timeout)
|
|
379
|
+
details.update(meta)
|
|
380
|
+
return ok, code, details, "http_fallback"
|
|
381
|
+
|
|
382
|
+
if mode == "playwright_fallback":
|
|
383
|
+
ok, code, meta = _playwright_probe(url, timeout)
|
|
384
|
+
details.update(meta)
|
|
385
|
+
if ok:
|
|
386
|
+
return ok, code, details, "playwright_fallback"
|
|
387
|
+
if chain:
|
|
388
|
+
ok, code, meta = _http_get_probe(url, timeout)
|
|
389
|
+
details.update(meta)
|
|
390
|
+
details["fallback_from"] = "playwright_fallback"
|
|
391
|
+
return ok, code, details, "http_fallback"
|
|
392
|
+
return ok, UAT_BROWSER_UNAVAILABLE, details, "playwright_fallback"
|
|
393
|
+
|
|
394
|
+
if dom_interaction and chain:
|
|
395
|
+
ok, code, meta = _playwright_probe(url, timeout)
|
|
396
|
+
if ok:
|
|
397
|
+
details.update(meta)
|
|
398
|
+
return ok, code, details, "playwright_fallback"
|
|
399
|
+
|
|
400
|
+
ok, code, meta = _http_get_probe(url, timeout)
|
|
401
|
+
details.update(meta)
|
|
402
|
+
if ok:
|
|
403
|
+
return ok, code, details, "http_fallback"
|
|
404
|
+
if chain and dom_interaction:
|
|
405
|
+
ok, code, meta = _playwright_probe(url, timeout)
|
|
406
|
+
details.update(meta)
|
|
407
|
+
details["fallback_from"] = "http_fallback"
|
|
408
|
+
return ok, code, details, "playwright_fallback"
|
|
409
|
+
return ok, code, details, "http_fallback"
|
|
410
|
+
|
|
411
|
+
|
|
412
|
+
def execute_browser_smoke(
|
|
413
|
+
step_text: str,
|
|
414
|
+
repo: Path,
|
|
415
|
+
*,
|
|
416
|
+
timeout: int = DEFAULT_PROBE_TIMEOUT,
|
|
417
|
+
) -> dict[str, object]:
|
|
418
|
+
mode = read_browser_probe_mode(repo)
|
|
419
|
+
url = resolve_browser_url(repo)
|
|
420
|
+
dom_interaction = _has_automatable_ui(step_text.lower())
|
|
421
|
+
result: dict[str, object] = {
|
|
422
|
+
"probe_kind": "browser_smoke",
|
|
423
|
+
"step": step_text[:200],
|
|
424
|
+
"stack_profile": detect_stack_profile(repo) or "unknown",
|
|
425
|
+
"probe_mode": mode,
|
|
426
|
+
"passed": False,
|
|
427
|
+
"reason_code": UAT_PROBE_UNRESOLVED,
|
|
428
|
+
"target_url": url,
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
if not url:
|
|
432
|
+
result["reason_code"] = UAT_PROBE_UNRESOLVED
|
|
433
|
+
return result
|
|
434
|
+
|
|
435
|
+
if mode == "cursor":
|
|
436
|
+
if is_mcp_unavailable():
|
|
437
|
+
result["reason_code"] = UAT_BROWSER_UNAVAILABLE
|
|
438
|
+
result["execution_tier"] = "stdlib"
|
|
439
|
+
if read_fallback_chain(repo):
|
|
440
|
+
ok, code, meta, fb_mode = _run_fallback_chain(
|
|
441
|
+
url, repo, timeout, dom_interaction=dom_interaction
|
|
442
|
+
)
|
|
443
|
+
result.update(meta)
|
|
444
|
+
result["probe_mode"] = fb_mode
|
|
445
|
+
result["passed"] = ok
|
|
446
|
+
result["reason_code"] = code if ok else UAT_BROWSER_PROBE_FAILED
|
|
447
|
+
if not ok and code == UAT_BROWSER_UNAVAILABLE:
|
|
448
|
+
result["reason_code"] = UAT_BROWSER_UNAVAILABLE
|
|
449
|
+
return result
|
|
450
|
+
result["execution_tier"] = "agent"
|
|
451
|
+
result["reason_code"] = UAT_PROBE_UNRESOLVED
|
|
452
|
+
result["agent_plan"] = {
|
|
453
|
+
"sequence": "browser_navigate → interact → screenshot → console/network → evidence",
|
|
454
|
+
"evidence_dir": "sprints/Sxxxx/evidence/browser/",
|
|
455
|
+
}
|
|
456
|
+
return result
|
|
457
|
+
|
|
458
|
+
result["execution_tier"] = "stdlib"
|
|
459
|
+
ok, code, meta, fb_mode = _run_fallback_chain(
|
|
460
|
+
url, repo, timeout, dom_interaction=dom_interaction
|
|
461
|
+
)
|
|
462
|
+
result.update(meta)
|
|
463
|
+
result["probe_mode"] = fb_mode
|
|
464
|
+
result["passed"] = ok
|
|
465
|
+
result["reason_code"] = code
|
|
466
|
+
return result
|
|
467
|
+
|
|
468
|
+
|
|
469
|
+
def execute_process_health(
|
|
470
|
+
step_text: str,
|
|
471
|
+
repo: Path,
|
|
472
|
+
*,
|
|
473
|
+
timeout: int = DEFAULT_PROBE_TIMEOUT,
|
|
474
|
+
poll_cap: int | None = None,
|
|
475
|
+
poll_interval: int | None = None,
|
|
476
|
+
) -> dict[str, object]:
|
|
477
|
+
cap, interval = read_poll_settings(repo)
|
|
478
|
+
if poll_cap is not None:
|
|
479
|
+
cap = poll_cap
|
|
480
|
+
if poll_interval is not None:
|
|
481
|
+
interval = poll_interval
|
|
482
|
+
|
|
483
|
+
startup = _extract_startup_command(step_text, repo)
|
|
484
|
+
health_url = _read_health_url(repo) or resolve_browser_url(repo)
|
|
485
|
+
result: dict[str, object] = {
|
|
486
|
+
"probe_kind": "process_health",
|
|
487
|
+
"step": step_text[:200],
|
|
488
|
+
"stack_profile": detect_stack_profile(repo) or "unknown",
|
|
489
|
+
"execution_tier": "stdlib",
|
|
490
|
+
"passed": False,
|
|
491
|
+
"reason_code": UAT_PROBE_UNRESOLVED,
|
|
492
|
+
}
|
|
493
|
+
|
|
494
|
+
if not startup or not health_url:
|
|
495
|
+
return result
|
|
496
|
+
|
|
497
|
+
result["startup_command"] = startup
|
|
498
|
+
result["health_url"] = health_url
|
|
499
|
+
proc: subprocess.Popen[str] | None = None
|
|
500
|
+
try:
|
|
501
|
+
proc = subprocess.Popen(
|
|
502
|
+
startup,
|
|
503
|
+
shell=True,
|
|
504
|
+
cwd=str(repo),
|
|
505
|
+
stdout=subprocess.DEVNULL,
|
|
506
|
+
stderr=subprocess.DEVNULL,
|
|
507
|
+
)
|
|
508
|
+
deadline = time.monotonic() + cap
|
|
509
|
+
while time.monotonic() < deadline:
|
|
510
|
+
ok, code, meta = _http_get_probe(health_url, min(timeout, interval + 2))
|
|
511
|
+
if ok:
|
|
512
|
+
result.update(meta)
|
|
513
|
+
result["passed"] = True
|
|
514
|
+
result["reason_code"] = UAT_PROBE_PASS
|
|
515
|
+
return result
|
|
516
|
+
time.sleep(interval)
|
|
517
|
+
result["reason_code"] = UAT_PROBE_TIMEOUT
|
|
518
|
+
except OSError as exc:
|
|
519
|
+
result["reason_code"] = UAT_PROBE_FAILED
|
|
520
|
+
result["error"] = str(exc)
|
|
521
|
+
finally:
|
|
522
|
+
if proc is not None:
|
|
523
|
+
proc.terminate()
|
|
524
|
+
try:
|
|
525
|
+
proc.wait(timeout=5)
|
|
526
|
+
except subprocess.TimeoutExpired:
|
|
527
|
+
proc.kill()
|
|
528
|
+
return result
|
|
529
|
+
|
|
530
|
+
|
|
531
|
+
def execute_cli_smoke(
|
|
532
|
+
step_text: str,
|
|
533
|
+
repo: Path,
|
|
534
|
+
*,
|
|
535
|
+
timeout: int = DEFAULT_PROBE_TIMEOUT,
|
|
536
|
+
) -> dict[str, object]:
|
|
537
|
+
cmd = _extract_backtick_command(step_text)
|
|
538
|
+
expect_out = _extract_stdout_expectation(step_text)
|
|
539
|
+
result: dict[str, object] = {
|
|
540
|
+
"probe_kind": "cli_smoke",
|
|
541
|
+
"step": step_text[:200],
|
|
542
|
+
"stack_profile": detect_stack_profile(repo) or "unknown",
|
|
543
|
+
"execution_tier": "stdlib",
|
|
544
|
+
"passed": False,
|
|
545
|
+
"reason_code": UAT_PROBE_UNRESOLVED,
|
|
546
|
+
}
|
|
547
|
+
if not cmd:
|
|
548
|
+
return result
|
|
549
|
+
result["command"] = cmd
|
|
550
|
+
try:
|
|
551
|
+
proc = subprocess.run(
|
|
552
|
+
cmd,
|
|
553
|
+
shell=True,
|
|
554
|
+
cwd=str(repo),
|
|
555
|
+
timeout=timeout,
|
|
556
|
+
capture_output=True,
|
|
557
|
+
text=True,
|
|
558
|
+
)
|
|
559
|
+
result["exit_code"] = proc.returncode
|
|
560
|
+
if proc.returncode != 0:
|
|
561
|
+
result["reason_code"] = UAT_PROBE_FAILED
|
|
562
|
+
return result
|
|
563
|
+
if expect_out and expect_out not in proc.stdout:
|
|
564
|
+
result["reason_code"] = UAT_PROBE_FAILED
|
|
565
|
+
return result
|
|
566
|
+
result["reason_code"] = UAT_PROBE_PASS
|
|
567
|
+
result["passed"] = True
|
|
568
|
+
except subprocess.TimeoutExpired:
|
|
569
|
+
result["reason_code"] = UAT_PROBE_TIMEOUT
|
|
570
|
+
except OSError as exc:
|
|
571
|
+
result["reason_code"] = UAT_PROBE_FAILED
|
|
572
|
+
result["error"] = str(exc)
|
|
573
|
+
return result
|
|
574
|
+
|
|
575
|
+
|
|
576
|
+
def validate_browser_evidence(result: dict[str, object], mode: str) -> dict[str, object]:
|
|
577
|
+
out = dict(result)
|
|
578
|
+
if not out.get("passed"):
|
|
579
|
+
return out
|
|
580
|
+
if mode != "cursor":
|
|
581
|
+
return out
|
|
582
|
+
refs = out.get("browser_evidence_refs")
|
|
583
|
+
if not isinstance(refs, dict):
|
|
584
|
+
out["passed"] = False
|
|
585
|
+
out["reason_code"] = UAT_BROWSER_PROBE_FAILED
|
|
586
|
+
return out
|
|
587
|
+
nav = str(refs.get("navigation_url") or "").strip()
|
|
588
|
+
screenshots = refs.get("screenshots")
|
|
589
|
+
if not isinstance(screenshots, list):
|
|
590
|
+
screenshots = []
|
|
591
|
+
console = refs.get("console_summary")
|
|
592
|
+
network = refs.get("network_summary")
|
|
593
|
+
console_path = ""
|
|
594
|
+
network_path = ""
|
|
595
|
+
if isinstance(console, dict):
|
|
596
|
+
console_path = str(console.get("summary_path") or "")
|
|
597
|
+
if isinstance(network, dict):
|
|
598
|
+
network_path = str(network.get("summary_path") or "")
|
|
599
|
+
has_evidence = bool(nav) and (
|
|
600
|
+
bool(screenshots) or bool(console_path) or bool(network_path)
|
|
601
|
+
)
|
|
602
|
+
if len(screenshots) > MAX_SCREENSHOTS:
|
|
603
|
+
has_evidence = False
|
|
604
|
+
if not has_evidence:
|
|
605
|
+
out["passed"] = False
|
|
606
|
+
out["reason_code"] = UAT_BROWSER_PROBE_FAILED
|
|
607
|
+
return out
|
|
608
|
+
|
|
609
|
+
|
|
610
|
+
def merge_result_fragment(fragment: dict[str, object], repo: Path) -> dict[str, object]:
|
|
611
|
+
mode = read_browser_probe_mode(repo)
|
|
612
|
+
if "probe_mode" not in fragment:
|
|
613
|
+
fragment = dict(fragment)
|
|
614
|
+
fragment["probe_mode"] = mode
|
|
615
|
+
return validate_browser_evidence(fragment, mode)
|
|
616
|
+
|
|
617
|
+
|
|
154
618
|
def execute_probe(
|
|
155
619
|
kind: str,
|
|
156
620
|
step_text: str,
|
|
@@ -176,6 +640,7 @@ def execute_probe(
|
|
|
176
640
|
if not cmd:
|
|
177
641
|
result["reason_code"] = UAT_PROBE_UNRESOLVED
|
|
178
642
|
return result
|
|
643
|
+
result["execution_tier"] = "stdlib"
|
|
179
644
|
return _run_subprocess(cmd, repo, timeout, result)
|
|
180
645
|
|
|
181
646
|
if kind == "test":
|
|
@@ -190,6 +655,7 @@ def execute_probe(
|
|
|
190
655
|
else:
|
|
191
656
|
result["reason_code"] = UAT_PROBE_UNRESOLVED
|
|
192
657
|
return result
|
|
658
|
+
result["execution_tier"] = "stdlib"
|
|
193
659
|
return _run_subprocess(cmd, repo, timeout, result)
|
|
194
660
|
|
|
195
661
|
if kind == "api_health":
|
|
@@ -197,24 +663,21 @@ def execute_probe(
|
|
|
197
663
|
if not url:
|
|
198
664
|
result["reason_code"] = UAT_PROBE_UNRESOLVED
|
|
199
665
|
return result
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
result["passed"] = 200 <= resp.status < 400
|
|
206
|
-
if not result["passed"]:
|
|
207
|
-
result["reason_code"] = UAT_PROBE_FAILED
|
|
208
|
-
except urllib.error.URLError as exc:
|
|
209
|
-
result["reason_code"] = UAT_PROBE_FAILED
|
|
210
|
-
result["error"] = type(exc).__name__
|
|
211
|
-
except TimeoutError:
|
|
212
|
-
result["reason_code"] = UAT_PROBE_TIMEOUT
|
|
666
|
+
result["execution_tier"] = "stdlib"
|
|
667
|
+
ok, code, meta = _http_get_probe(url, timeout)
|
|
668
|
+
result.update(meta)
|
|
669
|
+
result["passed"] = ok
|
|
670
|
+
result["reason_code"] = code if ok else UAT_PROBE_FAILED
|
|
213
671
|
return result
|
|
214
672
|
|
|
215
|
-
if kind
|
|
216
|
-
|
|
217
|
-
|
|
673
|
+
if kind == "process_health":
|
|
674
|
+
return execute_process_health(step_text, repo, timeout=timeout)
|
|
675
|
+
|
|
676
|
+
if kind == "browser_smoke":
|
|
677
|
+
return execute_browser_smoke(step_text, repo, timeout=timeout)
|
|
678
|
+
|
|
679
|
+
if kind == "cli_smoke":
|
|
680
|
+
return execute_cli_smoke(step_text, repo, timeout=timeout)
|
|
218
681
|
|
|
219
682
|
result["reason_code"] = UAT_PROBE_UNRESOLVED
|
|
220
683
|
return result
|
|
@@ -260,6 +723,12 @@ def resolve_and_probe(step_text: str, repo: Path) -> dict[str, object]:
|
|
|
260
723
|
}
|
|
261
724
|
if pre_reason == UAT_PROBE_FORBIDDEN:
|
|
262
725
|
return {"probe_kind": kind, "reason_code": UAT_PROBE_FORBIDDEN, "passed": False}
|
|
726
|
+
if pre_reason == UAT_PROBE_UNRESOLVED and kind in ("manual_operator",):
|
|
727
|
+
return {
|
|
728
|
+
"probe_kind": kind,
|
|
729
|
+
"reason_code": UAT_PROBE_UNRESOLVED,
|
|
730
|
+
"passed": False,
|
|
731
|
+
}
|
|
263
732
|
return execute_probe(kind, step_text, Path(repo))
|
|
264
733
|
|
|
265
734
|
|
|
@@ -269,25 +738,93 @@ def probe_steps(steps: list[str], repo: Path) -> list[dict[str, object]]:
|
|
|
269
738
|
|
|
270
739
|
def self_test() -> None:
|
|
271
740
|
repo = Path(__file__).resolve().parents[1]
|
|
272
|
-
assert UAT_PROBE_PASS in PROBE_KINDS or UAT_PROBE_PASS == "UAT_PROBE_PASS"
|
|
273
741
|
assert "build" in PROBE_KINDS
|
|
742
|
+
|
|
274
743
|
r = classify_step("run unit tests", repo)
|
|
275
744
|
assert r[0] == "test" or r[1] in (UAT_PROBE_UNRESOLVED, "")
|
|
745
|
+
|
|
276
746
|
r2 = classify_step("read secrets from .env file", repo)
|
|
277
747
|
assert r2[1] == UAT_PROBE_FORBIDDEN
|
|
278
|
-
|
|
279
|
-
|
|
748
|
+
|
|
749
|
+
r3 = classify_step("enter password in login form", repo)
|
|
750
|
+
assert r3[1] == UAT_PROBE_FORBIDDEN
|
|
751
|
+
|
|
752
|
+
r4 = classify_step("operator visually confirms button click", repo)
|
|
753
|
+
assert r4[0] == "manual_operator"
|
|
754
|
+
assert r4[1] == UAT_PROBE_UNRESOLVED
|
|
755
|
+
|
|
756
|
+
r5 = classify_step("click submit button on home page", repo)
|
|
757
|
+
assert r5[0] == "browser_smoke" or r5[1] == UAT_PROBE_UNRESOLVED
|
|
758
|
+
|
|
759
|
+
r6 = classify_step("operator verifies compliance manually", repo)
|
|
760
|
+
assert r6[0] == "manual_operator"
|
|
761
|
+
|
|
762
|
+
mode = read_browser_probe_mode(repo)
|
|
763
|
+
assert mode in BROWSER_PROBE_MODES
|
|
764
|
+
|
|
765
|
+
bs = execute_browser_smoke("browser smoke test ui", repo)
|
|
766
|
+
assert bs.get("probe_mode") == "cursor"
|
|
767
|
+
assert bs["passed"] is False
|
|
768
|
+
if bs.get("target_url"):
|
|
769
|
+
assert bs.get("execution_tier") == "agent"
|
|
770
|
+
assert bs["reason_code"] == UAT_PROBE_UNRESOLVED
|
|
771
|
+
else:
|
|
772
|
+
assert bs["reason_code"] == UAT_PROBE_UNRESOLVED
|
|
773
|
+
|
|
774
|
+
bad_pass = validate_browser_evidence(
|
|
775
|
+
{"passed": True, "reason_code": UAT_PROBE_PASS},
|
|
776
|
+
"cursor",
|
|
777
|
+
)
|
|
778
|
+
assert bad_pass["passed"] is False
|
|
779
|
+
assert bad_pass["reason_code"] == UAT_BROWSER_PROBE_FAILED
|
|
780
|
+
|
|
781
|
+
good_pass = validate_browser_evidence(
|
|
782
|
+
{
|
|
783
|
+
"passed": True,
|
|
784
|
+
"reason_code": UAT_PROBE_PASS,
|
|
785
|
+
"browser_evidence_refs": {
|
|
786
|
+
"navigation_url": "http://localhost:3000/",
|
|
787
|
+
"screenshots": ["sprints/S0082/evidence/browser/pr-01.png"],
|
|
788
|
+
},
|
|
789
|
+
},
|
|
790
|
+
"cursor",
|
|
791
|
+
)
|
|
792
|
+
assert good_pass["passed"] is True
|
|
793
|
+
|
|
794
|
+
cli = execute_cli_smoke('run CLI `python -c "import sys; sys.exit(0)"` exit code 0', repo)
|
|
795
|
+
assert cli["passed"] is True
|
|
796
|
+
assert cli["reason_code"] == UAT_PROBE_PASS
|
|
797
|
+
|
|
798
|
+
ph = execute_process_health(
|
|
799
|
+
"wait for server startup readiness",
|
|
800
|
+
repo,
|
|
801
|
+
poll_cap=1,
|
|
802
|
+
poll_interval=1,
|
|
803
|
+
)
|
|
804
|
+
assert ph["reason_code"] in (UAT_PROBE_UNRESOLVED, UAT_PROBE_TIMEOUT, UAT_PROBE_FAILED)
|
|
805
|
+
|
|
806
|
+
merged = merge_result_fragment(
|
|
807
|
+
{"passed": True, "probe_kind": "browser_smoke"},
|
|
808
|
+
repo,
|
|
809
|
+
)
|
|
810
|
+
assert merged["passed"] is False
|
|
811
|
+
|
|
280
812
|
assert detect_stack_profile(repo) in ("python", "node", None, "generated")
|
|
281
813
|
|
|
282
814
|
|
|
283
815
|
def main() -> int:
|
|
284
816
|
import argparse
|
|
285
817
|
|
|
286
|
-
parser = argparse.ArgumentParser(description="UAT probe resolver (US-0092).")
|
|
818
|
+
parser = argparse.ArgumentParser(description="UAT probe resolver (US-0092 / US-0093).")
|
|
287
819
|
parser.add_argument("--repo", default=".")
|
|
288
820
|
parser.add_argument("--step", action="append", default=[], help="Acceptance step text.")
|
|
289
821
|
parser.add_argument("--self-test", action="store_true")
|
|
290
822
|
parser.add_argument("--report", action="store_true", help="JSON probe results to stdout.")
|
|
823
|
+
parser.add_argument(
|
|
824
|
+
"--merge-result",
|
|
825
|
+
metavar="FRAGMENT.json",
|
|
826
|
+
help="Validate browser evidence fragment JSON (US-0093).",
|
|
827
|
+
)
|
|
291
828
|
args = parser.parse_args()
|
|
292
829
|
repo = Path(args.repo).resolve()
|
|
293
830
|
|
|
@@ -300,6 +837,20 @@ def main() -> int:
|
|
|
300
837
|
print("[UAT_PROBE_LIB_SELF_TEST_OK]")
|
|
301
838
|
return 0
|
|
302
839
|
|
|
840
|
+
if args.merge_result:
|
|
841
|
+
frag_path = Path(args.merge_result)
|
|
842
|
+
if not frag_path.is_file():
|
|
843
|
+
print(f"merge-result: file not found: {frag_path}", file=sys.stderr)
|
|
844
|
+
return 2
|
|
845
|
+
try:
|
|
846
|
+
fragment = json.loads(frag_path.read_text(encoding="utf-8"))
|
|
847
|
+
except json.JSONDecodeError as exc:
|
|
848
|
+
print(f"merge-result: invalid JSON: {exc}", file=sys.stderr)
|
|
849
|
+
return 2
|
|
850
|
+
validated = merge_result_fragment(fragment, repo)
|
|
851
|
+
print(json.dumps(validated, sort_keys=True, separators=(",", ":")))
|
|
852
|
+
return 0 if validated.get("passed") else 1
|
|
853
|
+
|
|
303
854
|
if args.report or args.step:
|
|
304
855
|
results = probe_steps(args.step or ["run tests"], repo)
|
|
305
856
|
print(json.dumps(results, sort_keys=True, separators=(",", ":")))
|