@capagents/forgeqe 2.3.15 → 2.4.0

package/py/forgeqe_engine/capture.py

@@ -18,7 +18,8 @@ from typing import Any, Dict, List, Optional
 
 from browser_use import ActionResult, BrowserSession
 
-from .memory import BASE, KEPT_ATTRS, candidate_locators, confidence, looks_generated
+from . import userlike
+from .memory import BASE, KEPT_ATTRS, candidate_locators, confidence, looks_generated, playwright_role_ok
 
 logger = logging.getLogger("forgeqe_engine.capture")
 
@@ -175,8 +176,9 @@ def install_effects(tools: Any, effects: List[Dict[str, Any]], captured: Dict[st
         before = await page_state(session) if name in EFFECT_ACTIONS and session is not None else None
         captured.pop("target", None)
         captured.pop("api", None)
+        userlike.reset()
         result = await original(action, browser_session, *args, **kwargs)
-        entry: Dict[str, Any] = {"action": name, "effect": None}
+        entry: Dict[str, Any] = {"action": name, "effect": None, **userlike.take()}
         if name in EFFECT_ACTIONS and session is not None and not getattr(result, "error", None):
             entry["effect"] = await effect_after(session, before)
         if captured.get("target") is not None:
@@ -236,7 +238,7 @@ _VERIFY_FN = r"""function(locs) {
       default: return l.css ? [...document.querySelectorAll(l.css)] : null;
     }
   };
-  return locs.map((l) => {
+  const results = locs.map((l) => {
     try {
       const all = find(l);
       if (!all) return null;
@@ -244,6 +246,8 @@ _VERIFY_FN = r"""function(locs) {
       return { count: hits.length, target: hits.length === 1 && (hits[0] === el || el.contains(hits[0])) };
     } catch (e) { return { count: -1, target: false }; }
   });
+  // What Playwright's getByRole reads for the element's role: its tag, href and role attribute.
+  return { results, el: { tag: el.tagName.toLowerCase(), href: el.getAttribute('href') || '', role: el.getAttribute('role') || '' } };
 }"""
 
 
@@ -272,14 +276,20 @@ async def verify_locators(session: Any, backend_node_id: Optional[int], locators
         object_id = await _object_for(cdp, backend_node_id)
         if not object_id:
             return locators
-        plain = [l for l in locators if l.get("type") != "role"]
+        plain = [l for l in locators if l.get("type") not in ("role", "role_pattern")]
         res = await _send(
             cdp, "Runtime", "callFunctionOn",
             {"objectId": object_id, "functionDeclaration": _VERIFY_FN, "arguments": [{"value": plain}], "returnByValue": True},
         )
-        results = dict(zip((id(l) for l in plain), ((res.get("result") or {}).get("value") or [])))
+        value = (res.get("result") or {}).get("value") or {}
+        el = value.get("el") or {}
+        results = dict(zip((id(l) for l in plain), value.get("results") or []))
         for loc in locators:
-            if loc.get("type") == "role":
+            if loc.get("type") in ("role", "role_pattern"):
+                if el and not playwright_role_ok(str(el.get("tag") or ""), {"href": el.get("href") or "", "role": el.get("role") or ""}, str(loc.get("role") or "")):
+                    # Chrome calls it that; Playwright would not find it by this role.
+                    loc["matches"], loc["verified"] = 0, False
+                    continue
                 try:
                     hits = await _ax_matches(cdp, str(loc.get("role") or ""), str(loc.get("name") or ""))
                 except Exception:
@@ -317,7 +327,23 @@ _FACTS_FN = r"""function(kept) {
   for (const k of kept) { const v = el.getAttribute(k); if (v) attrs[k] = v; }
   const xp = (e) => { const parts = []; for (; e && e.nodeType === 1; e = e.parentElement) { let i = 1; for (let s = e.previousElementSibling; s; s = s.previousElementSibling) if (s.tagName === e.tagName) i++; parts.unshift(e.tagName.toLowerCase() + '[' + i + ']'); } return parts.join('/'); };
   const classes = [...el.classList].filter((c) => /^[A-Za-z_-][\w-]*$/.test(c)).slice(0, 4);
-  return { tag: el.tagName.toLowerCase(), attributes: attrs, classes, xpath: xp(el), text: String(el.innerText || el.textContent || '').replace(/\s+/g, ' ').trim().slice(0, 120) };
+  let scope = '';
+  if (!el.id) for (let e = el.parentElement, d = 0; e && d < 6 && e !== document.body; e = e.parentElement, d++) {
+    if (e.id && !/\d{4,}|^(ember|mui-|radix-|react-select-|headlessui-|yui_|j_id|__)/i.test(e.id)) { scope = /^[A-Za-z_-][\w-]*$/.test(e.id) ? '#' + e.id : '[id="' + e.id.replace(/"/g, '\\"') + '"]'; break; }
+  }
+  // In a frame: what names it from the page that holds it (same origin), or at least its address.
+  let frame = null;
+  if (window !== window.top) {
+    let fe = null;
+    try { fe = window.frameElement; } catch (e) {}
+    frame = {};
+    if (fe) for (const k of ['src', 'id', 'name', 'title']) { const v = fe.getAttribute(k); if (v) frame[k] = v.slice(0, 300); }
+    if (!frame.src) frame.src = String(location.href).slice(0, 300);
+  }
+  const out = { tag: el.tagName.toLowerCase(), attributes: attrs, classes, xpath: xp(el), text: String(el.innerText || el.textContent || '').replace(/\s+/g, ' ').trim().slice(0, 120) };
+  if (scope) out.scope = scope;
+  if (frame) out.frame = frame;
+  return out;
 }"""
 
 

package/py/forgeqe_engine/logins.py

@@ -137,6 +137,11 @@ def _host_matches(host: str, login_hosts: List[str]) -> bool:
     return any(host == h or h.endswith("." + host) or host.endswith("." + h) for h in login_hosts)
 
 
+def can_sign_in(login: Login, missing: List[str]) -> bool:
+    """Whether the agent can sign in to this login's site itself: it has a username and a password to type."""
+    return all(env and env not in missing for env in (login.username_env, login.password_env))
+
+
 def load_sessions(logins: List[Login]) -> Optional[Dict[str, Any]]:
     """Cookies and local storage saved by earlier runs, merged into one storage state; None when there are none."""
     cookies: List[Dict[str, Any]] = []

package/py/forgeqe_engine/memory.py

@@ -61,6 +61,7 @@ BASE = {
     "title": 0.72,
     "text": 0.70,
     "css": 0.50,
+    "role_pattern": 0.80,
     "xpath": 0.30,
 }
 # Kinds precise enough to recognise the same element on a later visit; xpath is the last resort.
@@ -101,6 +102,37 @@ def _clean(text: Any, limit: int = 120) -> str:
     return re.sub(r"\s+", " ", str(text or "")).strip()[:limit]
 
 
+# Icon fonts (Font Awesome, Material Icons, Glyphicons) draw their icons with private-use characters from CSS. Chrome's
+# accessible name and the agent's labels include them; Playwright's getByText never sees CSS text, and a person
+# reading the handoff sees an empty name. They are kept out of names and text, and matched as "any non-word character".
+_ICONS = re.compile("[\ue000-\uf8ff\U000f0000-\U000ffffd\U00100000-\U0010fffd]")
+
+
+def strip_icons(text: Any) -> str:
+    return re.sub(r"\s+", " ", _ICONS.sub(" ", str(text or ""))).strip()
+
+
+def has_icons(text: Any) -> bool:
+    return bool(_ICONS.search(str(text or "")))
+
+
+def _js_regex(text: str) -> str:
+    return re.sub(r"[.*+?^${}()|[\]\\/]", lambda m: "\\" + m.group(0), text)
+
+
+def playwright_role_ok(tag: str, attrs: Dict[str, str], role: str) -> bool:
+    """
+    Whether Playwright's getByRole gives the element this role. Chrome's accessibility tree, which a role is read from,
+    makes a link of an <a> with no href (a click handler is enough); Playwright follows ARIA, where it is not one.
+    """
+    explicit = str(attrs.get("role") or "").split()
+    if explicit:
+        return explicit[0] == role
+    if tag in ("a", "area") and role == "link":
+        return bool(attrs.get("href"))
+    return True
+
+
 def _sha(text: str, n: int) -> str:
     return hashlib.sha1(text.encode("utf-8")).hexdigest()[:n]
 
@@ -187,6 +219,25 @@ def element_facts(node: Any) -> Dict[str, Any]:
         stable_hash = node.compute_stable_hash()
     except Exception:
         stable_hash = None
+    scope = ""
+    frame: Dict[str, str] = {}
+    try:
+        up, depth = getattr(node, "parent_node", None), 0
+        while up is not None and depth < 40:
+            up_tag = str(getattr(up, "node_name", "") or "").lower()
+            up_attrs = getattr(up, "attributes", None) or {}
+            # The nearest ancestor with an id names one item of a list (a cart row, a result card): a locator scoped to
+            # it is unique where the element's own classes repeat on every item.
+            if not scope and depth < 6 and not attrs.get("id"):
+                up_id = str(up_attrs.get("id") or "")
+                if up_id and not looks_generated(up_id):
+                    scope = f"#{up_id}" if _css_ident(up_id) else f"[id={_q(up_id)}]"
+            if up_tag in ("iframe", "frame"):
+                frame = {k: str(up_attrs[k])[:300] for k in ("src", "id", "name", "title") if up_attrs.get(k)}
+                break
+            up, depth = getattr(up, "parent_node", None), depth + 1
+    except Exception:
+        pass
     classes: List[str] = []
     cls = str(raw_attrs.get("class") or "")
     if cls:
@@ -206,6 +257,8 @@ def element_facts(node: Any) -> Dict[str, Any]:
         "classes": classes[:4],
         "xpath": xpath,
         "stable_hash": stable_hash,
+        **({"scope": scope} if scope else {}),
+        **({"frame": frame} if frame else {}),
     }
 
 
@@ -238,8 +291,14 @@ def candidate_locators(f: Dict[str, Any]) -> List[Dict[str, Any]]:
         v = a["id"]
         css = f"#{v}" if _css_ident(v) else f"[id={_q(v)}]"
         add("id", v, f"page.locator({_q(css)})", css=css)
-    if role in ROLES and name and len(name) <= 80:
-        add("role", f"{role}|{name}", f"page.getByRole({_q(role)}, {{ name: {_q(name)}, exact: true }})", role=role, name=name)
+    plain = strip_icons(name)
+    if role in ROLES and plain and len(name) <= 80 and playwright_role_ok(tag, a, role):
+        if plain == name:
+            add("role", f"{role}|{name}", f"page.getByRole({_q(role)}, {{ name: {_q(name)}, exact: true }})", role=role, name=name)
+        else:
+            # The name has an icon character: match its words, with any icon or space around them.
+            pattern = f"^\\W*{_js_regex(plain)}\\W*$"
+            add("role_pattern", f"{role}|{pattern}", f"page.getByRole({_q(role)}, {{ name: /{pattern}/ }})", role=role, name=name)
     if a.get("name"):
         css = f"{tag}[name={_q(a['name'])}]"
         add("name", a["name"], f"page.locator({_q(css)})", css=css)
@@ -259,13 +318,18 @@ def candidate_locators(f: Dict[str, Any]) -> List[Dict[str, Any]]:
     if tag == "a" and href and not href.startswith(("javascript:", "#")) and len(href) <= 200:
         css = f"a[href={_q(href)}]"
         add("href", href, f"page.locator({_q(css)})", css=css)
-    text = f.get("text") or (name if tag not in FORM_TAGS else "")
+    text = strip_icons(f.get("text") or (name if tag not in FORM_TAGS else ""))
     if text and len(text) <= 60 and tag not in FORM_TAGS:
         add("text", text, f"page.getByText({_q(text)}, {{ exact: true }})")
     classes = f.get("classes") or []
     if classes:
         css = tag + "".join(f".{c}" for c in classes[:2])
         add("css", css, f"page.locator({_q(css)})", css=css)
+    if f.get("scope"):
+        # Within the item it belongs to: one of a control that repeats on every row or card.
+        own = tag + "".join(f".{c}" for c in classes[:2]) if classes else (f"{tag}[href={_q(href)}]" if tag == "a" and href else tag)
+        css = f"{f['scope']} {own}"
+        add("css", css, f"page.locator({_q(css)})", css=css, scoped=True)
     if f.get("xpath"):
         xp = "/" + str(f["xpath"]).lstrip("/")
         add("xpath", xp, f"page.locator({_q('xpath=' + xp)})")
@@ -300,7 +364,15 @@ def _public_locator(loc: Dict[str, Any]) -> Dict[str, Any]:
 
 def element_label(facts: Dict[str, Any]) -> str:
     a = facts.get("attributes") or {}
-    return facts.get("name") or facts.get("text") or a.get("placeholder") or a.get("aria-label") or facts.get("tag") or ""
+    label = strip_icons(facts.get("name")) or strip_icons(facts.get("text")) or a.get("placeholder") or a.get("aria-label") or a.get("title") or ""
+    if label:
+        return label
+    tag = facts.get("tag") or ""
+    if has_icons(facts.get("name")) or has_icons(facts.get("text")):
+        # A control that shows only an icon: say so, with what names it in the page's code.
+        classes = facts.get("classes") or []
+        return f"icon-only {tag}{'.' + classes[0] if classes else ''}"
+    return tag
 
 
 # ---------------------------------------------------------------------------------------------
@@ -1084,6 +1156,10 @@ def compact_step(step: Dict[str, Any]) -> Dict[str, Any]:
         out["scripted"] = True
     if step.get("overlay"):
         out["overlay"] = step["overlay"]
+    # What the run met that a test must handle, and what it did in a way a test cannot repeat (see userlike).
+    for k in ("covered_by", "bypass", "frame"):
+        if step.get(k):
+            out[k] = step[k]
     if step.get("note"):
         out["note"] = step["note"]
     return out

package/py/forgeqe_engine/service.py

@@ -17,11 +17,12 @@ import time
 from dataclasses import dataclass, field
 from datetime import datetime, timezone
 from pathlib import Path
+from urllib.parse import urlparse
 from typing import Any, Callable, Dict, List, Optional, Set, Tuple
 
-from . import brand
+from . import brand, userlike
 from .diagnostics import Diagnostics, box_for
-from .logins import load_sessions, parse_logins, prompt_block, save_sessions, use_logins
+from .logins import _host_matches, can_sign_in, load_sessions, parse_logins, prompt_block, save_sessions, use_logins
 from .memory import READ_ONLY_ACTIONS, MemoryStore, compact_step, flow_step, page_id
 from .prompts import CONVENTIONS, text as prompt_text, use_prompts
 from .report import CASE_REPORT_INSTRUCTION, CaseReport, build_timeline, case_report_from_history
@@ -114,6 +115,11 @@ class RunRequest:
     memory: Dict[str, Any] = field(default_factory=dict)
     # [{name, domains, username_env, password_env, totp_env, session_file}]
     logins: List[Dict[str, Any]] = field(default_factory=list)
+    # Start the browser with the logins' saved sessions; false for a test case, which a test runs from a fresh browser.
+    restore_sessions: bool = True
+    # Set during the run: the logins whose saved session the browser started with, and each one's sites.
+    sessions_restored: List[str] = field(default_factory=list)
+    sessions_restored_hosts: Dict[str, List[str]] = field(default_factory=dict)
     # Number of steps in the test case this goal came from; the agent then reports a verdict for each.
     case_steps: int = 0
     # The test case and a key per step (changes when the step is reworded): an edited case replays its unchanged steps.
@@ -150,6 +156,7 @@ class RunRequest:
     def from_dict(cls, d: Dict[str, Any]) -> "RunRequest":
         return cls(
             logins=[x for x in (d.get("logins") or []) if isinstance(x, dict)],
+            restore_sessions=d.get("restore_sessions") is not False,
             case_steps=int(d["case_steps"]) if str(d.get("case_steps") or "").isdigit() else 0,
             case_id=str(d.get("case_id") or ""),
             case_step_keys=[str(k) for k in (d.get("case_step_keys") or []) if str(k)],
@@ -467,6 +474,8 @@ class MemoryRun:
     used_checkpoint: str = ""
     # Actions that worked without changing the page (dead clicks, read-only scripts): not part of the flow.
     no_effect: List[str] = field(default_factory=list)
+    # A click refused because something covered its element, by element key: kept for the click that then worked.
+    covered: Dict[str, Dict[str, str]] = field(default_factory=dict)
     # The test case this run is for, and a key per case step.
     case_id: str = ""
     case_keys: List[str] = field(default_factory=list)
@@ -808,6 +817,8 @@ def _record_memory_results(
         if target:
             mem.store.record_outcome(target["page"], target["key"], ok)
         if not ok:
+            if target and eff.get("covered_by"):
+                mem.covered[target["key"]] = eff["covered_by"]
             continue
         if eff.get("effect") is False and name in ("click", "send_keys", "evaluate", "click_selector"):
             mem.no_effect.append(line)
@@ -833,6 +844,12 @@ def _record_memory_results(
             step["effect"] = bool(eff["effect"])
         if name == "click" and target and target.get("overlay"):
             step["overlay"] = target["overlay"]
+        if target and target["key"] in mem.covered:
+            step["covered_by"] = mem.covered.pop(target["key"])
+        if eff.get("bypass"):
+            step["bypass"] = list(eff["bypass"])
+        if target and (target.get("facts") or {}).get("frame"):
+            step["frame"] = target["facts"]["frame"]
         if name == "evaluate" and eff.get("effect") is True:
             step["scripted"] = True
         if name == "api_request" and eff.get("api"):
@@ -956,11 +973,20 @@ def _write_artifacts(
     slim["start_url"] = start_url or ""
     slim["driver"] = "engine"
     slim["run_id"] = out_dir.name
+    used = sessions_used(req.sessions_restored_hosts, collected_steps)
+    if used:
+        slim["sessions_restored"] = used
     run_path.write_text(json.dumps(slim, indent=2, ensure_ascii=False, default=str), encoding="utf-8")
     files.insert(0, str(run_path.resolve()))
     return files
 
 
+def sessions_used(restored: Dict[str, List[str]], steps: List[Dict[str, Any]]) -> List[str]:
+    """The restored sessions of sites the run went to: what it started with that a test's browser does not have."""
+    visited = {(urlparse(str(s.get("url") or "")).hostname or "") for s in steps} - {""}
+    return [name for name, hosts in restored.items() if any(_host_matches(v, hosts) for v in visited)]
+
+
 async def run_browser_task(req: RunRequest, emit: Emit, cancel_event: asyncio.Event) -> Dict[str, Any]:
     """
     Run a single library Agent task and write artifacts into ``req.out_dir``.
@@ -994,6 +1020,7 @@ async def run_browser_task(req: RunRequest, emit: Emit, cancel_event: asyncio.Ev
 
     recorded_checks: List[Dict[str, Any]] = []
     brand.install()
+    userlike.install()
     headless = resolve_browser_headless(req.headed)
     timeout_sec = float(os.getenv("BROWSER_USE_TIMEOUT_SECONDS", "600"))
     step_timeout = int(os.getenv("BROWSER_USE_STEP_TIMEOUT", "180"))
@@ -1020,11 +1047,20 @@ async def run_browser_task(req: RunRequest, emit: Emit, cancel_event: asyncio.Ev
     sensitive, missing = use_logins(logins)
     for env in missing:
         emit({"type": "log", "level": "warn", "text": f"Saved login: {env} is not set, so that value is not available to the agent"})
-    stored = load_sessions(logins)
+    # A test case runs from the state its test starts from: signed out. A login the agent cannot sign in to itself (no
+    # username or password here) keeps its saved session, the only way in; the handoff then says the run started with it.
+    restoring = logins if req.restore_sessions else [l for l in logins if not can_sign_in(l, missing)]
+    stored = load_sessions(restoring)
     if stored:
         browser_kw["storage_state"] = stored
-        names = [l.name for l in logins if l.session_file and Path(l.session_file).is_file()]
-        emit({"type": "log", "level": "info", "text": f"Signed-in session restored for {', '.join(names)}"})
+        names = [l.name for l in restoring if l.session_file and Path(l.session_file).is_file()]
+        req.sessions_restored = names
+        req.sessions_restored_hosts = {l.name: l.hosts() for l in restoring if l.name in names}
+        why = "" if req.restore_sessions else " (no username and password to sign in with, so the run starts signed in; a test must sign in itself)"
+        emit({"type": "log", "level": "info", "text": f"Signed-in session restored for {', '.join(names)}{why}"})
+    skipped = [l.name for l in logins if l not in restoring and l.session_file and Path(l.session_file).is_file()]
+    if skipped:
+        emit({"type": "log", "level": "info", "text": f"Saved sessions not restored for {', '.join(skipped)}: a test case runs from the state its test starts from, and the agent signs in"})
     emit(
         {
             "type": "log",

package/py/forgeqe_engine/userlike.py

@@ -0,0 +1,150 @@
+"""
+The browser agent acts as a user can, or the run says where it did not.
+
+The library clicks an element that something else covers from JavaScript (element.click()), types to whatever has focus
+when it cannot reach a field, and clicks from JavaScript when the mouse click fails. A user cannot do any of these, and
+neither can a test: Playwright waits for the element to be the one under the mouse. So:
+
+- A click on a covered element fails, naming what covers it. The agent then closes or moves past it as a user would,
+  and that step is in the flow a test repeats.
+- What the library still does from a script is kept for the step it belongs to (take()), and the handoff says so.
+
+One engine process runs one browser run, so the record is per process.
+"""
+from __future__ import annotations
+
+import logging
+from typing import Any, Dict, List, Optional
+
+logger = logging.getLogger("forgeqe_engine.userlike")
+
+_SCRIPT_CLICK = "this.click()"
+
+_bypass: List[str] = []
+_covered: Optional[Dict[str, str]] = None
+
+# What covers the point: the element there, raised to the overlay or dialog it belongs to (the thing to close).
+_COVER_FN = r"""function(x, y) {
+  const at = document.elementFromPoint(x, y);
+  if (!at) return null;
+  let top = at;
+  for (let e = at; e && e !== document.body && e.nodeType === 1; e = e.parentElement) {
+    const cs = getComputedStyle(e);
+    const role = e.getAttribute('role');
+    if (e.tagName === 'DIALOG' || role === 'dialog' || role === 'alertdialog' || e.getAttribute('aria-modal') === 'true' || cs.position === 'fixed' || /\bmodal\b|overlay|popup|backdrop|consent|cookie/i.test(String(e.className || '') + ' ' + e.id)) top = e;
+  }
+  const cls = typeof top.className === 'string' ? top.className.trim().split(/\s+/).slice(0, 3).join('.') : '';
+  const tag = top.tagName.toLowerCase();
+  const html = '<' + tag + (top.id ? ' id="' + top.id + '"' : '') + (cls ? ' class="' + cls.replace(/\./g, ' ') + '"' : '') + '>';
+  const text = String(top.innerText || '').replace(/\s+/g, ' ').trim().slice(0, 120);
+  const frame = tag === 'iframe' ? String(top.getAttribute('src') || '').slice(0, 200) : '';
+  return { element: html, css: tag + (top.id ? '#' + top.id : cls ? '.' + cls : ''), text, frame };
+}"""
+
+
+class Covered(Exception):
+    """The element is under another one: a user's click lands on that one."""
+
+
+def reset() -> None:
+    global _covered
+    _bypass.clear()
+    _covered = None
+
+
+def take() -> Dict[str, Any]:
+    """What happened in the action since reset(): script actions, and what covered the element."""
+    out: Dict[str, Any] = {}
+    if _bypass:
+        out["bypass"] = list(dict.fromkeys(_bypass))
+    if _covered:
+        out["covered_by"] = dict(_covered)
+    reset()
+    return out
+
+
+def covered_message(cover: Dict[str, str]) -> str:
+    what = cover.get("element") or "another element"
+    words = f' ("{cover["text"][:80]}")' if cover.get("text") else ""
+    return (
+        f"The element is covered by {what}{words}, which would take the click: a user cannot click it now, and neither "
+        "can a test. Close or dismiss what covers it as a user would (its close, accept, or continue control; Escape for a "
+        "dialog), or wait for it to go, then click the element again."
+    )
+
+
+async def _what_covers(cdp_session: Any, x: float, y: float) -> Optional[Dict[str, str]]:
+    try:
+        doc = await cdp_session.cdp_client.send.Runtime.evaluate(
+            params={"expression": "document.documentElement", "returnByValue": False}, session_id=cdp_session.session_id
+        )
+        object_id = ((doc or {}).get("result") or {}).get("objectId")
+        if not object_id:
+            return None
+        res = await cdp_session.cdp_client.send.Runtime.callFunctionOn(
+            params={"objectId": object_id, "functionDeclaration": _COVER_FN, "arguments": [{"value": x}, {"value": y}], "returnByValue": True},
+            session_id=cdp_session.session_id,
+        )
+        value = ((res or {}).get("result") or {}).get("value")
+        if not isinstance(value, dict):
+            return None
+        from .memory import strip_icons
+
+        value["text"] = strip_icons(value.get("text"))
+        return value
+    except Exception as ex:
+        logger.debug("userlike: what covers the element not read: %s", ex)
+        return None
+
+
+def install() -> None:
+    """Patch the library once per process (see the module's description)."""
+    try:
+        from browser_use.browser.watchdogs.default_action_watchdog import DefaultActionWatchdog
+    except ImportError:
+        return
+    if getattr(DefaultActionWatchdog, "__forgeqe_userlike__", False):
+        return
+    DefaultActionWatchdog.__forgeqe_userlike__ = True  # type: ignore[attr-defined]
+
+    check = DefaultActionWatchdog._check_element_occlusion
+
+    async def occlusion(self: Any, backend_node_id: int, x: float, y: float, cdp_session: Any) -> bool:
+        global _covered
+        occluded = await check(self, backend_node_id, x, y, cdp_session)
+        if not occluded:
+            return False
+        cover = await _what_covers(cdp_session, x, y)
+        if not cover:
+            # Nothing to name (the element could not be resolved): the library clicks from a script, recorded below.
+            return True
+        _covered = cover
+        raise Covered(covered_message(cover))
+
+    DefaultActionWatchdog._check_element_occlusion = occlusion  # type: ignore[method-assign]
+
+    type_to_page = getattr(DefaultActionWatchdog, "_type_to_page", None)
+    if type_to_page is not None:
+
+        async def typed(self: Any, *args: Any, **kwargs: Any) -> Any:
+            _bypass.append("typed to the page (into whatever had focus), because the field itself could not be reached")
+            return await type_to_page(self, *args, **kwargs)
+
+        DefaultActionWatchdog._type_to_page = typed  # type: ignore[method-assign]
+
+    try:
+        from cdp_use.cdp.runtime.library import RuntimeClient
+    except ImportError:
+        return
+    call = RuntimeClient.callFunctionOn
+
+    async def call_function_on(self: Any, params: Any = None, *args: Any, **kwargs: Any) -> Any:
+        try:
+            code = str((params or {}).get("functionDeclaration") or "")
+            if _SCRIPT_CLICK in code:
+                _bypass.append("clicked it from a script (element.click()), not with the mouse")
+        except Exception:
+            pass
+        return await call(self, params, *args, **kwargs)
+
+    RuntimeClient.callFunctionOn = call_function_on  # type: ignore[method-assign]

package/py/forgeqe_mobile/hyperexecute.py

@@ -165,7 +165,7 @@ def write_project(project: Path, files: Sequence[Tuple[Path, str]], tests: Seque
         "framework": {"name": "raw", "args": args},
         "env": {"MAESTRO": True, "MAESTRO_LOGS_DIR": "MaestroLogs", "MAESTRO_CLI_NO_ANALYTICS": "1"},
         "pre": ["chmod +x ./setup.sh ./run.sh", "./setup.sh"],
-        "testDiscovery": {"value": names, "mode": "static", "type": "raw"},
+        "testDiscovery": {"command": "echo ForgeQE", "value": names, "mode": "static", "type": "raw"},
         "testRunnerCommand": "./run.sh $test",
         "frameworkStatusOnly": True,
         "report": True,