@capagents/forgeqe 0.0.0-stage → 1.0.0

package/py/forgeqe_engine/capture.py

@@ -0,0 +1,538 @@
+"""
+Live capture for the test a passed run becomes, done while the page is still there:
+
+- what each action changed on the page (an action that changed nothing is not part of the path),
+- the control a user acts on, not the decorative node under the pointer,
+- which locators find exactly that element on the live page (role and name through Chrome's accessibility tree,
+  the same names Playwright's getByRole uses; CSS, ids, and XPath in the page),
+- the element and section behind each recorded check,
+- click_selector, a real mouse click on an element found by CSS, so the agent never has to click from JavaScript.
+"""
+# No `from __future__ import annotations`: the library matches the injected browser_session parameter by its real type.
+import asyncio
+import json
+import logging
+import time
+from typing import Any, Dict, List, Optional
+
+from browser_use import ActionResult, BrowserSession
+
+from .memory import BASE, KEPT_ATTRS, candidate_locators, confidence, looks_generated
+
+logger = logging.getLogger("forgeqe_engine.capture")
+
+# Actions whose only evidence of working is a change on the page; fills and selects always count.
+EFFECT_ACTIONS = frozenset({"click", "send_keys", "evaluate", "click_selector"})
+EFFECT_WAIT_SECONDS = 1.5
+CDP_TIMEOUT = 3.0
+
+ACTIONABLE_TAGS = frozenset({"a", "button", "input", "select", "textarea", "summary", "option", "label"})
+ACTIONABLE_ROLES = frozenset(
+    {"button", "link", "checkbox", "radio", "switch", "tab", "menuitem", "menuitemcheckbox", "menuitemradio", "option",
+     "combobox", "textbox", "searchbox", "treeitem", "slider", "spinbutton"}
+)
+MAX_CLIMB = 5
+
+CAPTURE_SYSTEM_MESSAGE = """
+## Acting so the run can become a test
+A passed run is turned into an automated test that repeats your actions with real clicks and typing. Act only through the browser actions (click, input, select_dropdown, send_keys, click_selector). Never click, submit, navigate, or change values from evaluate (element.click(), dispatchEvent, location.href = …, setting .value): a test cannot repeat that. Use evaluate only to read the page. Open by URL only the start page or a URL the task gives: adding #section or ?parameters to a URL to land where a step would take you skips that step, and a test cannot count it as done. If you open a wrong URL, go back to the start page and do the steps. When the control you need has no index, find it with find_elements or evaluate, then click it with click_selector and a CSS selector that matches only that element.
+""".strip()
+
+# A cheap fingerprint of what a user could notice: address, visible text, structure, open menus and dialogs, form state.
+_STATE_JS = r"""(() => {
+  const hash = (s) => { let h = 0; for (let i = 0; i < s.length; i++) h = (h * 31 + s.charCodeAt(i)) | 0; return h };
+  const body = document.body;
+  const text = (body && body.innerText) || '';
+  const visible = (e) => e.getClientRects().length > 0;
+  const open = [...document.querySelectorAll('[aria-expanded="true"], dialog[open], [role="dialog"], [role="menu"], [role="listbox"], [aria-modal="true"]')].filter(visible).length;
+  const fields = [...document.querySelectorAll('input, textarea, select')];
+  const values = fields.map((e) => (e.type === 'checkbox' || e.type === 'radio') ? (e.checked ? '1' : '0') : String(e.value || '')).join('\u0001');
+  const selected = [...document.querySelectorAll('[aria-selected="true"], [aria-checked="true"], [aria-pressed="true"]')].length;
+  return { url: location.href, text: hash(text), len: text.length, nodes: document.getElementsByTagName('*').length, open, values: hash(values), selected };
+})()"""
+
+
+async def _cdp(session: Any):
+    return await asyncio.wait_for(session.get_or_create_cdp_session(), timeout=CDP_TIMEOUT)
+
+
+async def _send(cdp: Any, domain: str, method: str, params: Dict[str, Any]) -> Dict[str, Any]:
+    fn = getattr(getattr(cdp.cdp_client.send, domain), method)
+    return await asyncio.wait_for(fn(params=params, session_id=cdp.session_id), timeout=CDP_TIMEOUT)
+
+
+async def page_state(session: Any) -> Optional[Dict[str, Any]]:
+    """The page's fingerprint, or None when the page could not be asked (navigating, closed)."""
+    try:
+        cdp = await _cdp(session)
+        res = await _send(cdp, "Runtime", "evaluate", {"expression": _STATE_JS, "returnByValue": True})
+        value = (res.get("result") or {}).get("value")
+        return value if isinstance(value, dict) else None
+    except Exception as ex:
+        logger.debug("capture: page state unavailable: %s", ex)
+        return None
+
+
+def page_changed(before: Optional[Dict[str, Any]], after: Optional[Dict[str, Any]]) -> Optional[bool]:
+    """True when anything a user could notice changed; None when either state is unknown."""
+    if not before or not after:
+        return None
+    return any(before.get(k) != after.get(k) for k in ("url", "text", "len", "nodes", "open", "values", "selected"))
+
+
+async def effect_after(session: Any, before: Optional[Dict[str, Any]], wait: float = EFFECT_WAIT_SECONDS) -> Optional[bool]:
+    """Whether the page changed since `before`, polling briefly for menus, results, and navigations that come late."""
+    if before is None:
+        return None
+    deadline = time.monotonic() + wait
+    delay = 0.0
+    while True:
+        if delay:
+            await asyncio.sleep(delay)
+        after = await page_state(session)
+        if after is None:
+            return True if delay else None  # the page went away mid-poll: most likely it navigated
+        if page_changed(before, after):
+            return True
+        if time.monotonic() >= deadline:
+            return False
+        delay = min(0.5, max(0.15, delay * 2 or 0.15))
+
+
+def action_name(action: Any) -> str:
+    try:
+        data = action.model_dump(exclude_unset=True)
+    except Exception:
+        return ""
+    return next((k for k, v in data.items() if v is not None), "")
+
+
+def install_effects(tools: Any, effects: List[Dict[str, Any]], captured: Dict[str, Any]) -> None:
+    """
+    Wrap tools.act so each action call leaves {action, effect} in `effects`, in call order; click_selector also leaves
+    the element it clicked (from `captured`). Effects are measured only for actions whose result is a change on the page.
+    """
+    original = tools.act
+
+    async def act(action: Any, browser_session: Any = None, *args: Any, **kwargs: Any):
+        session = browser_session if browser_session is not None else kwargs.get("browser_session")
+        name = action_name(action)
+        before = await page_state(session) if name in EFFECT_ACTIONS and session is not None else None
+        captured.pop("target", None)
+        result = await original(action, browser_session, *args, **kwargs)
+        entry: Dict[str, Any] = {"action": name, "effect": None}
+        if name in EFFECT_ACTIONS and session is not None and not getattr(result, "error", None):
+            entry["effect"] = await effect_after(session, before)
+        if captured.get("target") is not None:
+            entry["target"] = captured.pop("target")
+        effects.append(entry)
+        return result
+
+    tools.act = act
+
+
+# ---------------------------------------------------------------------------------------------
+# The element a user acts on, and locators verified on the live page
+# ---------------------------------------------------------------------------------------------
+
+
+def _actionable(node: Any) -> bool:
+    tag = str(getattr(node, "node_name", "") or "").lower()
+    attrs = getattr(node, "attributes", None) or {}
+    if tag == "a":
+        return bool(attrs.get("href")) or str(attrs.get("role") or "") in ACTIONABLE_ROLES
+    if tag in ACTIONABLE_TAGS:
+        return True
+    role = str(attrs.get("role") or getattr(getattr(node, "ax_node", None), "role", "") or "")
+    return role in ACTIONABLE_ROLES
+
+
+def actionable_node(node: Any) -> Any:
+    """The node itself when a user acts on it, else its nearest ancestor that is a control (an icon inside a button → the button)."""
+    current, depth = node, 0
+    while current is not None and depth <= MAX_CLIMB:
+        if _actionable(current):
+            return current
+        current, depth = getattr(current, "parent_node", None), depth + 1
+    return node
+
+
+# Count what each CSS/XPath/text/attribute locator matches, and whether it is the element (or inside it: a click there reaches it).
+_VERIFY_FN = r"""function(locs) {
+  const el = this;
+  const norm = (s) => String(s || '').replace(/\s+/g, ' ').trim();
+  const q = (v) => '"' + String(v).replace(/\\/g, '\\\\').replace(/"/g, '\\"') + '"';
+  const visible = (e) => e.getClientRects().length > 0;
+  const byText = (t) => [...document.querySelectorAll('body *')].filter((e) => visible(e) && norm(e.textContent) === t && ![...e.children].some((c) => norm(c.textContent) === t));
+  const find = (l) => {
+    switch (l.type) {
+      case 'xpath': {
+        const r = document.evaluate(l.value, document, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
+        return Array.from({ length: r.snapshotLength }, (_, i) => r.snapshotItem(i));
+      }
+      case 'text': return byText(norm(l.value));
+      case 'placeholder': return [...document.querySelectorAll('[placeholder=' + q(l.value) + ']')];
+      case 'alt': return [...document.querySelectorAll('[alt=' + q(l.value) + ']')];
+      case 'title': return [...document.querySelectorAll('[title=' + q(l.value) + ']')];
+      case 'test_id': return [...document.querySelectorAll('[' + (l.attr || 'data-testid') + '=' + q(l.value) + ']')];
+      default: return l.css ? [...document.querySelectorAll(l.css)] : null;
+    }
+  };
+  return locs.map((l) => {
+    try {
+      const all = find(l);
+      if (!all) return null;
+      const hits = all.filter(visible).length ? all.filter(visible) : all;
+      return { count: hits.length, target: hits.length === 1 && (hits[0] === el || el.contains(hits[0])) };
+    } catch (e) { return { count: -1, target: false }; }
+  });
+}"""
+
+
+async def _object_for(cdp: Any, backend_node_id: int) -> Optional[str]:
+    res = await _send(cdp, "DOM", "resolveNode", {"backendNodeId": backend_node_id})
+    return ((res or {}).get("object") or {}).get("objectId")
+
+
+async def _ax_matches(cdp: Any, role: str, name: str) -> List[int]:
+    """backend node ids of the elements Chrome's accessibility tree names `name` with `role`."""
+    doc = await _send(cdp, "DOM", "getDocument", {"depth": 0})
+    root = ((doc or {}).get("root") or {}).get("backendNodeId")
+    res = await _send(cdp, "Accessibility", "queryAXTree", {"backendNodeId": root, "accessibleName": name, "role": role})
+    return [n["backendDOMNodeId"] for n in (res or {}).get("nodes") or [] if n.get("backendDOMNodeId") and not n.get("ignored")]
+
+
+async def verify_locators(session: Any, backend_node_id: Optional[int], locators: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+    """
+    Each locator with `matches` (elements it finds on the live page) and `verified` (exactly one, and it is this
+    element or inside it). Locators that could not be checked are returned unchanged. Verified ones come first.
+    """
+    if not backend_node_id or not locators:
+        return locators
+    try:
+        cdp = await _cdp(session)
+        object_id = await _object_for(cdp, backend_node_id)
+        if not object_id:
+            return locators
+        plain = [l for l in locators if l.get("type") != "role"]
+        res = await _send(
+            cdp, "Runtime", "callFunctionOn",
+            {"objectId": object_id, "functionDeclaration": _VERIFY_FN, "arguments": [{"value": plain}], "returnByValue": True},
+        )
+        results = dict(zip((id(l) for l in plain), ((res.get("result") or {}).get("value") or [])))
+        for loc in locators:
+            if loc.get("type") == "role":
+                try:
+                    hits = await _ax_matches(cdp, str(loc.get("role") or ""), str(loc.get("name") or ""))
+                except Exception:
+                    continue
+                inside = len(hits) == 1 and (hits[0] == backend_node_id or await _contains(cdp, object_id, hits[0]))
+                loc["matches"], loc["verified"] = len(hits), bool(inside)
+                continue
+            r = results.get(id(loc))
+            if isinstance(r, dict) and r.get("count", -1) >= 0:
+                loc["matches"], loc["verified"] = int(r["count"]), bool(r.get("target"))
+    except Exception as ex:
+        logger.debug("capture: locator check failed: %s", ex)
+        return locators
+    for loc in locators:
+        if "verified" in loc:
+            loc["confidence"] = confidence(loc)
+    return sorted(locators, key=lambda l: (not l.get("verified"), -int(l.get("confidence") or 0), -BASE.get(str(l.get("type")), 0)))
+
+
+async def _contains(cdp: Any, object_id: str, backend_node_id: int) -> bool:
+    other = await _object_for(cdp, backend_node_id)
+    if not other:
+        return False
+    res = await _send(
+        cdp, "Runtime", "callFunctionOn",
+        {"objectId": object_id, "functionDeclaration": "function(o){return this.contains(o)}", "arguments": [{"objectId": other}], "returnByValue": True},
+    )
+    return bool((res.get("result") or {}).get("value"))
+
+
+# Facts (as memory.element_facts gives them) for an element reached through CDP rather than the agent's element list.
+_FACTS_FN = r"""function(kept) {
+  const el = this;
+  const attrs = {};
+  for (const k of kept) { const v = el.getAttribute(k); if (v) attrs[k] = v; }
+  const xp = (e) => { const parts = []; for (; e && e.nodeType === 1; e = e.parentElement) { let i = 1; for (let s = e.previousElementSibling; s; s = s.previousElementSibling) if (s.tagName === e.tagName) i++; parts.unshift(e.tagName.toLowerCase() + '[' + i + ']'); } return parts.join('/'); };
+  const classes = [...el.classList].filter((c) => /^[A-Za-z_-][\w-]*$/.test(c)).slice(0, 4);
+  return { tag: el.tagName.toLowerCase(), attributes: attrs, classes, xpath: xp(el), text: String(el.innerText || el.textContent || '').replace(/\s+/g, ' ').trim().slice(0, 120) };
+}"""
+
+
+async def element_facts_cdp(session: Any, backend_node_id: int) -> Optional[Dict[str, Any]]:
+    try:
+        cdp = await _cdp(session)
+        object_id = await _object_for(cdp, backend_node_id)
+        if not object_id:
+            return None
+        res = await _send(
+            cdp, "Runtime", "callFunctionOn",
+            {"objectId": object_id, "functionDeclaration": _FACTS_FN, "arguments": [{"value": list(KEPT_ATTRS)}], "returnByValue": True},
+        )
+        facts = (res.get("result") or {}).get("value") or {}
+        # Build-generated class names (b87c397a13, css-1x2y3z) change with the site's next deploy.
+        facts["classes"] = [c for c in facts.get("classes") or [] if not looks_generated(c)]
+        ax = await _send(cdp, "Accessibility", "getPartialAXTree", {"backendNodeId": backend_node_id, "fetchRelatives": False})
+        node = next(iter((ax or {}).get("nodes") or []), {})
+        role = str(((node.get("role") or {}).get("value")) or "")
+        name = " ".join(str(((node.get("name") or {}).get("value")) or "").split())[:120]
+        # As memory.element_facts: the accessible name when there is one, the element's text only without it.
+        facts.update(role=facts["attributes"].get("role") or role, name=name, stable_hash=None, text="" if name else facts.get("text", ""))
+        return facts
+    except Exception as ex:
+        logger.debug("capture: element facts failed: %s", ex)
+        return None
+
+
+async def capture_element(session: Any, backend_node_id: int, facts: Optional[Dict[str, Any]] = None) -> Optional[Dict[str, Any]]:
+    """{facts, locators (verified first)} for one element on the live page."""
+    facts = facts or await element_facts_cdp(session, backend_node_id)
+    if not facts:
+        return None
+    locs = candidate_locators(facts)
+    for loc in locs:
+        loc.setdefault("matches", 1)
+        loc["confidence"] = confidence(loc)
+    return {"facts": facts, "locators": await verify_locators(session, backend_node_id, locs)}
+
+
+async def flow_target(session: Any, node: Any, target: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
+    """
+    The memory target for an action, re-pointed at the control a user acts on, with its locators checked on the live
+    page. The memory element (and its score) stays the one the agent targeted.
+    """
+    if target is None or node is None:
+        return target
+    from .memory import element_facts
+
+    control = actionable_node(node)
+    try:
+        facts = element_facts(control) if control is not node else target["facts"]
+    except Exception:
+        facts = target["facts"]
+    locs = candidate_locators(facts) if control is not node else [dict(l) for l in target.get("locators") or []]
+    if control is not node:
+        for loc in locs:
+            loc.setdefault("matches", 1)
+            loc["confidence"] = confidence(loc)
+    checked = await verify_locators(session, getattr(control, "backend_node_id", None), locs)
+    out = dict(target, facts=facts, locators=checked)
+    overlay = await overlay_of(session, getattr(control, "backend_node_id", None))
+    if overlay:
+        out["overlay"] = overlay
+    out["current"] = {(l["type"], l["value"]): l.get("matches", 1) for l in checked if l.get("verified")} or target.get("current") or {}
+    if control is not node:
+        out["climbed"] = True
+    return out
+
+
+# ---------------------------------------------------------------------------------------------
+# Overlays: the dialog or banner an action's element sits in
+# ---------------------------------------------------------------------------------------------
+
+# A dialog, or a fixed bar or box that reads like a cookie, consent, sign-in, or newsletter prompt. A sticky site
+# header is not one: it stays while the user works, so its controls are part of the flow.
+_OVERLAY_FN = r"""function() {
+  const PROMPT = /cookie|consent|privacy|gdpr|onetrust|sign[ -]?in|log[ -]?in|subscribe|newsletter|notification|genius|app/i;
+  const kind = (e) => {
+    const role = e.getAttribute('role');
+    if (e.tagName === 'DIALOG' || role === 'dialog' || role === 'alertdialog' || e.getAttribute('aria-modal') === 'true') return 'dialog';
+    const cs = getComputedStyle(e);
+    if (cs.position !== 'fixed') return null;
+    const hint = (e.id + ' ' + (typeof e.className === 'string' ? e.className : '') + ' ' + (e.getAttribute('aria-label') || '') + ' ' + String(e.innerText || '').slice(0, 300));
+    return PROMPT.test(hint) ? 'banner' : null;
+  };
+  for (let e = this; e && e !== document.body && e.nodeType === 1; e = e.parentElement) {
+    const k = kind(e);
+    if (!k) continue;
+    const heading = e.querySelector('h1, h2, h3, [role=heading]');
+    const name = (e.getAttribute('aria-label') || (heading && heading.textContent) || '').replace(/\s+/g, ' ').trim().slice(0, 80);
+    const text = (e.id + ' ' + (typeof e.className === 'string' ? e.className : '') + ' ' + name + ' ' + String(e.innerText || '').slice(0, 400)).toLowerCase();
+    const what = /cookie|consent|gdpr|onetrust|privacy/.test(text) ? 'cookie banner' : /sign[ -]?in|log[ -]?in|account/.test(text) ? 'sign-in prompt' : k === 'dialog' ? 'dialog' : 'banner';
+    return { kind: what, name, id: e.id || '' };
+  }
+  return null;
+}"""
+
+
+async def overlay_of(session: Any, backend_node_id: Optional[int]) -> Optional[Dict[str, Any]]:
+    """The overlay (dialog, cookie banner, sign-in prompt) the element sits in, or None for an element of the page itself."""
+    if not backend_node_id:
+        return None
+    try:
+        cdp = await _cdp(session)
+        object_id = await _object_for(cdp, backend_node_id)
+        if not object_id:
+            return None
+        res = await _send(cdp, "Runtime", "callFunctionOn", {"objectId": object_id, "functionDeclaration": _OVERLAY_FN, "returnByValue": True})
+        value = (res.get("result") or {}).get("value")
+        return value if isinstance(value, dict) else None
+    except Exception as ex:
+        logger.debug("capture: overlay check failed: %s", ex)
+        return None
+
+
+# ---------------------------------------------------------------------------------------------
+# The element behind a check
+# ---------------------------------------------------------------------------------------------
+
+# The element showing the text: the smallest visible one, raised to a near ancestor with a test id or id (a short
+# wrapper a test can name). When it sits in an item that repeats on the page (a result card, a row), the item's
+# selector, how many there are, and the element's selector inside one item: what a check about "every item" needs.
+_LOCATE_TEXT_JS = r"""(function(want){
+  const norm = (s) => String(s || '').replace(/\s+/g, ' ').trim().toLowerCase();
+  const w = norm(want);
+  const visible = (e) => e.getClientRects().length > 0;
+  const TID = ['data-testid', 'data-test-id', 'data-test', 'data-qa', 'data-cy'];
+  const tidOf = (e) => { for (const a of TID) { const v = e.getAttribute(a); if (v) return [a, v]; } return null; };
+  const q = (v) => '"' + String(v).replace(/\\/g, '\\\\').replace(/"/g, '\\"') + '"';
+  const sel = (e) => { const t = tidOf(e); return t ? '[' + t[0] + '=' + q(t[1]) + ']' : null; };
+  let best = null;
+  for (const e of document.querySelectorAll('body *')) {
+    if (!visible(e) || ['SCRIPT', 'STYLE', 'NOSCRIPT'].includes(e.tagName)) continue;
+    if (!norm(e.innerText).includes(w)) continue;
+    if ([...e.children].some((c) => visible(c) && norm(c.innerText).includes(w))) continue;
+    if (!best || norm(e.innerText).length < norm(best.innerText).length) best = e;
+  }
+  if (!best) return null;
+  // A heading, link, control, or cell already names itself; only a bare wrapper (div, span, …) is raised.
+  const NAMED = ['H1', 'H2', 'H3', 'H4', 'H5', 'H6', 'A', 'BUTTON', 'LABEL', 'INPUT', 'SELECT', 'TEXTAREA', 'TD', 'TH', 'LI', 'OPTION'];
+  const BLOCK = ['SECTION', 'ARTICLE', 'MAIN', 'ASIDE', 'NAV', 'FORM', 'TABLE', 'UL', 'OL', 'DIALOG', 'HEADER', 'FOOTER'];
+  let el = best;
+  if (!NAMED.includes(best.tagName) && !tidOf(best) && !best.id) {
+    for (let e = best.parentElement, i = 1; e && e !== document.body && i <= 3; e = e.parentElement, i++) {
+      if (BLOCK.includes(e.tagName) || norm(e.innerText).length > Math.max(3 * w.length, w.length + 40)) break;
+      if (tidOf(e) || e.id) { el = e; break; }
+    }
+  }
+  let item = null, itemSelector = null, itemCount = 0;
+  for (let e = el.parentElement; e && e !== document.body; e = e.parentElement) {
+    const s = sel(e);
+    if (!s) continue;
+    const n = document.querySelectorAll(s).length;
+    if (n > 1) { item = e; itemSelector = s; itemCount = n; break; }
+  }
+  let inside = null;
+  if (item) {
+    const s = sel(el);
+    if (s && item.querySelectorAll(s).length === 1) inside = s;
+  }
+  const section = el.closest('section, article, main, aside, nav, form, table, dialog, [role=region], [role=dialog], [role=main], [id]:not(body):not(html)');
+  return {
+    el,
+    section: !item && section && section !== el && section.contains(el) && norm(section.innerText).length < 5000 ? section : null,
+    item: itemSelector ? { selector: itemSelector, count: itemCount, inside } : null,
+  };
+})"""
+
+# Locators that spell out the text a check found: wrong for live data, which differs on the next run.
+_VALUE_LOCATORS = frozenset({"text", "role", "label", "title", "alt", "xpath"})
+
+
+async def locate_check(session: Any, text: str, dynamic: bool = False) -> Optional[Dict[str, Any]]:
+    """
+    The element showing `text`, with locators checked on the live page, and either its section or the repeating item
+    it belongs to (with the item's selector, count, and the element's selector inside one item). For live data the
+    element's locators that spell out this run's text are left out.
+    """
+    try:
+        cdp = await _cdp(session)
+        res = await _send(
+            cdp, "Runtime", "evaluate",
+            {"expression": f"{_LOCATE_TEXT_JS}({json.dumps(text)})", "returnByValue": False},
+        )
+        obj = (res.get("result") or {}).get("objectId")
+        if not obj:
+            return None
+        props = await _send(cdp, "Runtime", "getProperties", {"objectId": obj, "ownProperties": True})
+        ids: Dict[str, int] = {}
+        item: Optional[Dict[str, Any]] = None
+        for p in (props or {}).get("result") or []:
+            oid = (p.get("value") or {}).get("objectId")
+            if p.get("name") in ("el", "section") and oid:
+                desc = await _send(cdp, "DOM", "describeNode", {"objectId": oid})
+                backend = ((desc or {}).get("node") or {}).get("backendNodeId")
+                if backend:
+                    ids[p["name"]] = int(backend)
+            elif p.get("name") == "item" and oid:
+                val = await _send(cdp, "Runtime", "callFunctionOn", {"objectId": oid, "functionDeclaration": "function(){return this}", "returnByValue": True})
+                item = (val.get("result") or {}).get("value") or None
+        if "el" not in ids:
+            return None
+        element = await capture_element(session, ids["el"])
+        if not element:
+            return None
+        if dynamic:
+            stable = [l for l in element["locators"] if l.get("type") not in _VALUE_LOCATORS]
+            element["locators"] = stable
+        out: Dict[str, Any] = {"element": element}
+        if item:
+            out["item"] = item
+        if "section" in ids:
+            section = await capture_element(session, ids["section"])
+            if section:
+                out["section"] = section
+        return out
+    except Exception as ex:
+        logger.debug("capture: check element not located: %s", ex)
+        return None
+
+
+def public_locators(locs: List[Dict[str, Any]], limit: int = 4) -> List[Dict[str, Any]]:
+    keep = ("type", "value", "playwright", "confidence", "verified", "matches", "role", "name", "css", "attr")
+    chosen = [l for l in locs if l.get("verified")] or locs
+    return [{k: l[k] for k in keep if k in l} for l in chosen[:limit]]
+
+
+# ---------------------------------------------------------------------------------------------
+# click_selector
+# ---------------------------------------------------------------------------------------------
+
+
+def register_click_selector(tools: Any, captured: Dict[str, Any]) -> None:
+    """A real mouse click on the one visible element a CSS selector matches; the element is captured for the test."""
+
+    @tools.action(
+        "Click the element that a CSS selector matches, with a real mouse click. Use it when the control you need has no "
+        "index; the selector must match exactly one visible element. Never click from evaluate instead."
+    )
+    async def click_selector(selector: str, browser_session: BrowserSession) -> ActionResult:
+        try:
+            cdp = await _cdp(browser_session)
+            doc = await _send(cdp, "DOM", "getDocument", {"depth": 0})
+            found = await _send(cdp, "DOM", "querySelectorAll", {"nodeId": doc["root"]["nodeId"], "selector": selector})
+            node_ids = (found or {}).get("nodeIds") or []
+            boxes = []
+            for nid in node_ids[:20]:
+                try:
+                    quads = await _send(cdp, "DOM", "getContentQuads", {"nodeId": nid})
+                except Exception:
+                    continue
+                if (quads or {}).get("quads"):
+                    boxes.append(nid)
+            if len(boxes) != 1:
+                return ActionResult(error=f"{selector!r} matches {len(boxes)} visible elements; give a selector that matches exactly one.")
+            nid = boxes[0]
+            await _send(cdp, "DOM", "scrollIntoViewIfNeeded", {"nodeId": nid})
+            quad = (await _send(cdp, "DOM", "getContentQuads", {"nodeId": nid}))["quads"][0]
+            x, y = sum(quad[0::2]) / 4, sum(quad[1::2]) / 4
+            desc = await _send(cdp, "DOM", "describeNode", {"nodeId": nid})
+            backend = int(desc["node"]["backendNodeId"])
+            hit = await _send(cdp, "DOM", "getNodeForLocation", {"x": int(x), "y": int(y), "includeUserAgentShadowDOM": False})
+            if hit.get("backendNodeId") and hit["backendNodeId"] != backend:
+                obj = await _object_for(cdp, backend)
+                if obj and not await _contains(cdp, obj, int(hit["backendNodeId"])):
+                    return ActionResult(error=f"{selector!r} is covered by another element at its centre; close what covers it first.")
+            for kind in ("mouseMoved", "mousePressed", "mouseReleased"):
+                params: Dict[str, Any] = {"type": kind, "x": x, "y": y}
+                if kind != "mouseMoved":
+                    params.update(button="left", clickCount=1)
+                await _send(cdp, "Input", "dispatchMouseEvent", params)
+            captured["target"] = await capture_element(browser_session, backend)
+            return ActionResult(extracted_content=f"Clicked {selector}", long_term_memory=f"Clicked {selector}")
+        except Exception as ex:
+            return ActionResult(error=f"click_selector failed: {ex}")

package/py/forgeqe_engine/checks.py

@@ -0,0 +1,98 @@
+"""
+What the browser agent checked: each expected result, and the exact text on the page that shows it.
+
+A passed run becomes a test, and the test asserts these checks; ForgeQE's Playwright replay finds each text on
+the final page and hands the coding agent a locator that matched there.
+"""
+import json
+import logging
+from typing import Any, Dict, List, Optional
+
+from browser_use import ActionResult, BrowserSession
+
+from .capture import locate_check, public_locators
+
+logger = logging.getLogger(__name__)
+
+CHECK_ACTION = "record_check"
+
+CHECKS_SYSTEM_MESSAGE = """
+## Recording checks (required before done with success)
+A passed run becomes an automated test that asserts what you checked. For each result the task asks you to verify, call record_check once before done: `expected` is that result in a few words, and `text` is the shortest exact text on the page that shows it (a heading, a value, a label, the chosen option), copied as the page shows it. Set `dynamic` to true when that text is live data that will differ on another run (today's date, a price, a count, today's featured item), so the test asserts the rule rather than this value. Record only what the task asks you to verify.
+When the task lists expected results, record exactly one check per expected result, with `expected` copied from that line. ForgeQE finds the element showing `text`, so pick the text that points at what the line is about:
+- A result about the items in a list (how many there are, what every item shows): copy text from inside one item, such as one result's name or its rating, never a summary line like "2,463 properties found". ForgeQE then finds the repeating item and the field in it.
+- A page or section that opens: its heading or another label that names it.
+
+## When an expected result does not hold
+This is a test: an expected result that does not hold is a valid result, the test failing, not a mistake for you to work around. Do every step of the task first, each through the page's own controls as the step says.
+Settle each expected result with one read of the element it is about (the infobox row, the heading, the result card): when that element shows something else, record holds=false straight away. Do not try other spellings, number formats, or searches for the expected value. When the page does not show what an expected result says (another value, another text), record it once with record_check holds=false, text being exactly what the page shows instead, then go on with the remaining steps and expected results. Do not search the page again for the expected value, look for it elsewhere, or redo steps to make it hold. Finish with done success=false, starting with "Expected result not met:" and naming each one with what the page showed. Use success=false for the task itself only when a step could not be done.
+""".strip()
+
+# Visible text, plus the values of form fields, which visible text leaves out.
+_SHOWS_JS = """(function(want){
+  const norm = (s) => String(s || '').replace(/\\s+/g, ' ').trim().toLowerCase();
+  const w = norm(want);
+  if (!w) return false;
+  if (norm(document.body && document.body.innerText).includes(w)) return true;
+  return [...document.querySelectorAll('input, textarea, select')].some((e) => norm(e.value).includes(w));
+})(%s)"""
+
+
+async def page_shows(browser_session: BrowserSession, text: str) -> Optional[bool]:
+    """Whether the current page shows `text`; None when the page could not be asked."""
+    try:
+        cdp = await browser_session.get_or_create_cdp_session()
+        result = await cdp.cdp_client.send.Runtime.evaluate(
+            params={"expression": _SHOWS_JS % json.dumps(text), "returnByValue": True},
+            session_id=cdp.session_id,
+        )
+        value = (result.get("result") or {}).get("value")
+        return value if isinstance(value, bool) else None
+    except Exception as ex:
+        logger.debug("record_check: could not read the page: %s", ex)
+        return None
+
+
+def register_checks(tools: Any, checks: List[Dict[str, Any]]) -> None:
+    """Add record_check to the agent's tools; each check it records lands in `checks`."""
+
+    @tools.action(
+        "Record one result the task asks you to verify, before done: expected (the result in a few words), "
+        "text (the shortest exact text on the page that shows it, copied as shown), and dynamic=true when that text "
+        "is live data that will differ on another run (today's date, a price, a count, today's featured item). "
+        "holds=false when the page does not show the expected result: then text is what the page shows instead."
+    )
+    async def record_check(expected: str, text: str, browser_session: BrowserSession, dynamic: bool = False, holds: bool = True) -> ActionResult:
+        expected, text = " ".join(str(expected or "").split())[:200], " ".join(str(text or "").split())[:200]
+        if not text:
+            return ActionResult(error="Give the text on the page that shows the result.")
+        if await page_shows(browser_session, text) is False:
+            return ActionResult(error=f"The page does not show {text!r}. Record the text exactly as the page shows it.")
+        url = ""
+        try:
+            url = await browser_session.get_current_page_url()
+        except Exception:
+            pass
+        check: Dict[str, Any] = {"expected": expected or text, "text": text, "url": url}
+        if dynamic:
+            check["dynamic"] = True
+        if not holds:
+            # A failed expectation is a test result: `text` is what the page shows instead.
+            check["holds"] = False
+        # The element showing the text and its section, with locators checked on this page, for the test's assertion.
+        found = await locate_check(browser_session, text, dynamic=dynamic)
+        if found:
+            el = found["element"]
+            check["element"] = {"tag": el["facts"].get("tag"), "role": el["facts"].get("role"), "name": el["facts"].get("name"), "locators": public_locators(el["locators"])}
+            if found.get("item"):
+                check["item"] = found["item"]
+            if found.get("section"):
+                sec = found["section"]
+                check["container"] = {"tag": sec["facts"].get("tag"), "name": sec["facts"].get("name"), "locators": public_locators(sec["locators"], 2)}
+        checks[:] = [c for c in checks if c["expected"].lower() != expected.lower()]
+        checks.append(check)
+        if not holds:
+            return ActionResult(
+                extracted_content=f"Recorded: expected result not met: {expected} (the page shows {text!r}). That is this test's result; carry on with the remaining steps and checks, then finish."
+            )
+        return ActionResult(extracted_content=f"Recorded check: {expected or text} ({text!r})")