@capagents/forgeqe 1.2.1 → 1.3.1

package/py/forgeqe_engine/observe.py

@@ -0,0 +1,123 @@
+"""
+Replaying a flow without the model, and knowing it still passes (docs/COST_ARCHITECTURE.md, phase 2).
+
+Two parts:
+- read_only_script: whether an `evaluate` step in a flow only read the page. The agent often runs a script to look at
+  the page (list the fields, read their values) before it decides; replay can leave such a step out. The test is on
+  the script's source and conservative: anything that could write (assign, call a method that changes the page or the
+  browser's state, send a request, schedule code) counts as writing, and the step then needs the agent as before.
+- checks_for_flow / flow_checks_hold: the results a passing run recorded (record_check), kept in its flow with the
+  step each was recorded after, and checked again on the replayed page with no model.
+"""
+from __future__ import annotations
+
+import asyncio
+import re
+from typing import Any, Awaitable, Callable, Dict, List, Optional
+
+# Anything that writes to the page, the browser, or the network, or runs code later. A script with any of these
+# is not read-only. False positives only cost a model call (today's path); a false negative could skip a real step,
+# so the list leans wide.
+_WRITES = re.compile(
+    r"""
+    (?<![=!<>])=(?![=>])                                   # any assignment (=, +=, -=, ...), not ==, ===, =>, <=, >=, !=
+    | \+\+ | --                                            # increments
+    | \b(?:delete)\b
+    | \.(?:click|submit|requestSubmit|reset|focus|blur|select|setSelectionRange|setRangeText|showModal|show|close|open|
+         dispatchEvent|remove|removeChild|append|appendChild|prepend|insertBefore|insertAdjacent\w*|replaceChild|
+         replaceWith|replaceChildren|before|after|setAttribute\w*|removeAttribute\w*|toggleAttribute|
+         scroll\w*|play|pause|load|reload|assign|pushState|replaceState|back|forward|go|
+         setItem|removeItem|clear|add|toggle|set|setProperty|removeProperty|attachShadow|
+         sendBeacon|postMessage|writeText|write|writeln|execCommand|requestFullscreen|exitFullscreen)\s*\(
+    | \b(?:fetch|XMLHttpRequest|WebSocket|EventSource|eval|Function|setTimeout|setInterval|requestAnimationFrame|
+         queueMicrotask|importScripts|import)\s*\(
+    | \bnew\s+(?:\w*Event|XMLHttpRequest|WebSocket|Worker|SharedWorker|Function|MutationObserver|BroadcastChannel)\b
+    | \b(?:document\.cookie|location|indexedDB|caches|navigator\.serviceWorker)\b
+    """,
+    re.VERBOSE,
+)
+
+# Declarations (const x = ..., let x = ..., var x = ..., function parameters with defaults) write only local names.
+_DECLARATION = re.compile(r"\b(?:const|let|var)\s+(?:[A-Za-z_$][\w$]*|\{[^}]*\}|\[[^\]]*\])\s*=(?![=>])")
+# Text inside string and template literals: an "=" or ".click(" in a selector or a message is not code.
+_STRINGS = re.compile(r"""'(?:\\.|[^'\\])*'|"(?:\\.|[^"\\])*"|`(?:\\.|[^`\\])*`""", re.DOTALL)
+# Comments.
+_COMMENTS = re.compile(r"//[^\n]*|/\*.*?\*/", re.DOTALL)
+
+
+def read_only_script(code: str) -> bool:
+    """Whether a script only reads the page: no assignment to anything but its own new local names, no call that changes
+    the page, the browser, or storage, no request, and no code run later."""
+    src = str(code or "")
+    if not src.strip() or len(src) > 20_000:
+        return False
+    src = _COMMENTS.sub(" ", src)
+    src = _STRINGS.sub('""', src)
+    # `const x = ...`: the assignment is the declaration of a local name. Remove the "=" of each declaration only;
+    # the expression after it is still checked.
+    src = _DECLARATION.sub(lambda m: m.group(0)[:-1] + " ", src)
+    # Parameters with defaults, `(a = 1) =>`: rare in these scripts; treated as an assignment, so not read-only.
+    return _WRITES.search(src) is None
+
+
+def read_only_step(step: Dict[str, Any]) -> bool:
+    """A flow step replay can leave out: an `evaluate` whose script only reads the page."""
+    return step.get("action") == "evaluate" and read_only_script(str((step.get("params") or {}).get("code") or ""))
+
+
+# ---------------------------------------------------------------------------------------------
+# Recorded checks
+# ---------------------------------------------------------------------------------------------
+
+CHECK_ATTEMPTS = 6
+CHECK_DELAY_SECONDS = 0.5
+
+
+def checks_for_flow(checks: List[Dict[str, Any]], at: Dict[str, int]) -> List[Dict[str, Any]]:
+    """The checks a passing run recorded, as its flow keeps them: what to find on the page, and after which flow step."""
+    out: List[Dict[str, Any]] = []
+    for c in checks or []:
+        text = " ".join(str(c.get("text") or "").split())
+        if not text:
+            continue
+        entry: Dict[str, Any] = {"expected": str(c.get("expected") or text)[:200], "text": text[:200], "at": int(at.get(check_key(c), 0))}
+        if c.get("url"):
+            entry["url"] = str(c["url"])
+        if c.get("dynamic"):
+            entry["dynamic"] = True
+        if c.get("holds") is False:
+            entry["holds"] = False
+        if isinstance(c.get("step"), int) and c["step"] > 0:
+            entry["step"] = c["step"]
+        out.append(entry)
+    return out
+
+
+def check_key(c: Dict[str, Any]) -> str:
+    return str(c.get("expected") or c.get("text") or "").strip().lower()
+
+
+def checkable(flow: Dict[str, Any]) -> Optional[str]:
+    """None when the flow's checks can decide a replay on their own; otherwise why not, in words."""
+    checks = flow.get("checks")
+    if not isinstance(checks, list) or not checks:
+        return "the flow has no recorded checks"
+    if any(c.get("dynamic") for c in checks):
+        return "a recorded check is live data (dynamic), which only the agent can judge"
+    if any(c.get("holds") is False for c in checks):
+        return "a recorded check did not hold when it was recorded"
+    return None
+
+
+async def check_holds(shows: Callable[[str], Awaitable[Optional[bool]]], text: str, attempts: Optional[int] = None, delay: Optional[float] = None) -> Optional[bool]:
+    """Whether the page shows `text`, asking again for a few seconds while it settles; None when it could not be asked."""
+    attempts = CHECK_ATTEMPTS if attempts is None else attempts
+    delay = CHECK_DELAY_SECONDS if delay is None else delay
+    seen: Optional[bool] = None
+    for n in range(max(1, attempts)):
+        seen = await shows(text)
+        if seen:
+            return True
+        if n + 1 < attempts:
+            await asyncio.sleep(delay)
+    return seen

package/py/forgeqe_engine/replay.py

@@ -16,6 +16,7 @@ from typing import Any, Awaitable, Callable, Dict, List, Optional, Tuple
 
 from . import brand
 from .logins import totp_code
+from .observe import check_holds, read_only_step
 from .memory import (
     ELEMENT_ACTIONS,
     READ_ONLY_ACTIONS,
@@ -45,6 +46,10 @@ class ReplayOutcome:
     times: List[Tuple[float, float]] = field(default_factory=list)
     # Steps whose recorded locator no longer matched and another one found the element.
     heals: List[Dict[str, Any]] = field(default_factory=list)
+    # The flow's recorded checks found on the page again, each after the step it was recorded after (observe.py).
+    checked: List[Dict[str, Any]] = field(default_factory=list)
+    # Steps left out because they only read the page (an `evaluate` with a read-only script).
+    skipped_reads: int = 0
 
     @property
     def complete(self) -> bool:
@@ -86,7 +91,13 @@ async def replay_flow(
     cancel_event: asyncio.Event,
     on_step: Optional[Callable[[], Awaitable[None]]] = None,
     overlay: Any = None,
+    checks_first: bool = False,
 ) -> ReplayOutcome:
+    """
+    `checks_first` (registry.mode on, docs/COST_ARCHITECTURE.md): steps that only read the page are left out, and the
+    flow's recorded checks are found on the page again after the step each was recorded after. A check that does not
+    hold stops the replay there, so the agent takes over on that page as after any other stop.
+    """
     from browser_use import Tools
 
     tools = Tools()
@@ -99,6 +110,23 @@ async def replay_flow(
         out.reason = reason
         return out
 
+    checks = [c for c in (flow.get("checks") or []) if isinstance(c, dict) and c.get("text")] if checks_first else []
+
+    async def verify_checks(after: int) -> Optional[str]:
+        """Finds again the checks recorded after flow step `after`; the first that does not hold, in words."""
+        due = [c for c in checks if (int(c.get("at") or 0) == after) or (after == len(steps) and not 0 < int(c.get("at") or 0) <= len(steps))]
+        if not due:
+            return None
+        from .checks import page_shows
+
+        for c in due:
+            holds = await check_holds(lambda t: page_shows(browser, t), str(c["text"]))
+            out.checked.append({**c, "holds_now": bool(holds)})
+            emit({"type": "result", "step": after, "items": [{"ok": bool(holds), "done": False, "text": f'check "{c.get("expected") or c["text"]}": {"shows" if holds else "does not show"} "{c["text"]}"'[:240]}]})
+            if not holds:
+                return f'the recorded check "{c.get("expected") or c["text"]}" does not hold: the page does not show "{c["text"]}"'
+        return None
+
     for n, step in enumerate(steps, 1):
         if cancel_event.is_set():
             stop(n, "stopped by user")
@@ -106,6 +134,17 @@ async def replay_flow(
         action = str(step.get("action") or "")
         params = dict(step.get("params") or {})
         line = str(step.get("line") or action)
+        if checks_first and read_only_step(step):
+            # The agent looked at the page here; replay has nothing to do. Kept in the flow so check positions hold.
+            out.done.append(step)
+            out.times.append((time.time(), time.time()))
+            out.emitted = n
+            out.skipped_reads += 1
+            failed = await verify_checks(n)
+            if failed:
+                stop(n + 1 if n < len(steps) else n, failed)
+                break
+            continue
         if action not in REPLAYABLE_ACTIONS:
             stop(n, f"`{line}` needs the agent")
             break
@@ -188,6 +227,10 @@ async def replay_flow(
                 out.heals.append({"step": n, "line": line, "label": step.get("label") or "", "stale": [_locator_text(k) for k in gone], "used": _locator_text(used)})
         if on_step is not None:
             await on_step()
+        failed = await verify_checks(n)
+        if failed:
+            stop(n + 1 if n < len(steps) else n, failed)
+            break
 
     out.url = await _current_url(browser)
     return out
@@ -197,11 +240,14 @@ def handoff_task(goal: str, flow: Dict[str, Any], outcome: ReplayOutcome, *, kin
     """
     The agent's task after a replay: the goal, what already happened, and where to pick up.
 
-    `kind` is "exact" (an earlier successful run of this task), "resume" (the steps that worked in a run of this
+    `kind` is "exact" (an earlier successful run of this task), "prefix" (the recorded steps of a test case's unchanged
+    first steps, after the case was edited), "resume" (the steps that worked in a run of this
     task that later went wrong), or "interrupted" (the steps a run of this task got through before it was cut off: closed, crashed, or the machine restarted).
     """
     when = str(flow.get("last_success") or flow.get("at") or "")[:10]
-    if kind == "resume":
+    if kind == "prefix":
+        source = "an earlier successful run of this test case, before its later steps were changed"
+    elif kind == "resume":
         source = "an earlier run of this task that went wrong later"
     elif kind == "interrupted":
         source = "an earlier run of this task that was cut off before it finished"
@@ -221,6 +267,13 @@ def handoff_task(goal: str, flow: Dict[str, Any], outcome: ReplayOutcome, *, kin
             f"All recorded steps ran and the browser is on {outcome.url or 'the final page'}. Check this page: "
             "if the task is complete, call done right away and report the result. Otherwise continue from here."
         )
+    elif kind == "prefix":
+        n = int(flow.get("case_steps_done") or 0)
+        lines.append(
+            f"They did test steps 1 to {n}, which have not changed. The browser is on {outcome.url or 'the current page'}. "
+            f"Check that the page agrees, then continue with test step {n + 1}. In your step report, report steps 1 to {n} "
+            "from what the page shows now."
+        )
     elif kind == "interrupted":
         lines.append(
             f"That is as far as the earlier run got before it was cut off. The browser is on {outcome.url or 'the current page'}. "

package/py/forgeqe_engine/service.py

@@ -15,6 +15,7 @@ import os
 import re
 import time
 from dataclasses import dataclass, field
+from datetime import datetime, timezone
 from pathlib import Path
 from typing import Any, Callable, Dict, List, Optional, Set, Tuple
 
@@ -25,6 +26,7 @@ from .memory import READ_ONLY_ACTIONS, MemoryStore, compact_step, flow_step, pag
 from .prompts import CONVENTIONS, text as prompt_text, use_prompts
 from .report import CASE_REPORT_INSTRUCTION, CaseReport, build_timeline, case_report_from_history
 from .replay import ReplayOutcome, handoff_task, replay_flow
+from .observe import check_key, checkable, checks_for_flow
 from .settings import (
     _truthy_env,
     _browser_use_agent_tuning_kwargs,
@@ -107,6 +109,9 @@ class RunRequest:
     logins: List[Dict[str, Any]] = field(default_factory=list)
     # Number of steps in the test case this goal came from; the agent then reports a verdict for each.
     case_steps: int = 0
+    # The test case and a key per step (changes when the step is reworded): an edited case replays its unchanged steps.
+    case_id: str = ""
+    case_step_keys: List[str] = field(default_factory=list)
     # Step screenshots for the report: "all", "failures", or "off".
     screenshots: str = "all"
     # Accessibility checks on each page visited, for the report.
@@ -139,6 +144,8 @@ class RunRequest:
         return cls(
             logins=[x for x in (d.get("logins") or []) if isinstance(x, dict)],
             case_steps=int(d["case_steps"]) if str(d.get("case_steps") or "").isdigit() else 0,
+            case_id=str(d.get("case_id") or ""),
+            case_step_keys=[str(k) for k in (d.get("case_step_keys") or []) if str(k)],
             screenshots=str(d.get("screenshots") or "all").lower(),
             accessibility=d.get("accessibility") is not False,
             overlay=d.get("overlay") is not False,
@@ -395,6 +402,41 @@ def serialize_browser_step_payload(browser_state: Any, agent_output: Any, n_step
 
 
 REPLAY_MODES = ("verify", "trust", "off")
+# checks_first (docs/COST_ARCHITECTURE.md): every Nth replay of a flow is still checked by the agent, so a check that
+# passes when it should not is caught; and a flow not passed for this long is checked by the agent again.
+CANARY_EVERY = 10
+MAX_FLOW_AGE_DAYS = 30
+
+
+def _int(raw: Any, default: int) -> int:
+    try:
+        n = int(raw)
+    except (TypeError, ValueError):
+        return default
+    return n if n >= 0 else default
+
+
+def judged_by_checks(flow: Dict[str, Any], replay: Any, mem: "MemoryRun") -> Optional[Dict[str, Any]]:
+    """
+    After a complete replay of an exact flow: whether its recorded checks decide the run with no model. ok: every
+    check was found on the page again. Otherwise `why` says what leaves it to the agent (today's verify path).
+    """
+    why = checkable(flow)
+    if why:
+        return {"ok": False, "why": why}
+    if len(replay.checked) < len(flow.get("checks") or []) or not all(c.get("holds_now") for c in replay.checked):
+        return {"ok": False, "why": "not every recorded check was found on the page again"}
+    replays = int(flow.get("replays") or 0) + 1
+    if mem.canary_every and replays % mem.canary_every == 0:
+        return {"ok": False, "why": f"replay {replays} of this flow: every {mem.canary_every}th is checked by the agent too"}
+    last = str(flow.get("last_success") or "")
+    try:
+        age = (datetime.now(timezone.utc) - datetime.fromisoformat(last.replace("Z", "+00:00"))).days if last else None
+    except ValueError:
+        age = None
+    if age is None or age > mem.max_age_days:
+        return {"ok": False, "why": f"the flow last passed more than {mem.max_age_days} days ago" if age is not None else "the flow has no recorded pass"}
+    return {"ok": True, "why": ""}
 
 
 @dataclass
@@ -418,6 +460,59 @@ class MemoryRun:
     used_checkpoint: str = ""
     # Actions that worked without changing the page (dead clicks, read-only scripts): not part of the flow.
     no_effect: List[str] = field(default_factory=list)
+    # The test case this run is for, and a key per case step.
+    case_id: str = ""
+    case_keys: List[str] = field(default_factory=list)
+    # Case step number -> flow steps done when its expected result was recorded (the end of that step).
+    check_at: Dict[int, int] = field(default_factory=dict)
+    # Ends of the case steps a replay already did, carried over from the flow that was replayed.
+    carried_ends: List[Optional[int]] = field(default_factory=list)
+    # registry.mode on (docs/COST_ARCHITECTURE.md): the flow keeps its checks and a replay is judged by them.
+    checks_first: bool = False
+    # The checks the replayed flow had, and those the agent recorded this run with the flow step each came after.
+    flow_checks: List[Dict[str, Any]] = field(default_factory=list)
+    recorded: List[Dict[str, Any]] = field(default_factory=list)
+    check_pos: Dict[str, int] = field(default_factory=dict)
+    check_text: Dict[str, Any] = field(default_factory=dict)
+    canary_every: int = CANARY_EVERY
+    max_age_days: int = MAX_FLOW_AGE_DAYS
+
+    def checks_to_save(self, status: str) -> Optional[List[Dict[str, Any]]]:
+        """The checks the flow keeps after this run; None leaves the flow's as they are."""
+        if not self.checks_first or status != "complete":
+            return None
+        new = checks_for_flow(self.recorded, self.check_pos)
+        if not self.llm_steps:
+            # A replay judged by its checks: the same steps, so the same checks.
+            return list(self.flow_checks) if self.flow_checks else None
+        # Checks from the replayed part still stand where the agent did not record them again.
+        done = len(self.replay.done) if self.replay else 0
+        keys = {check_key(c) for c in new}
+        carried = [c for c in self.flow_checks if 0 < int(c.get("at") or 0) <= done and check_key(c) not in keys]
+        return carried + new
+
+    def note_checks(self, checks: List[Dict[str, Any]]) -> None:
+        """After an agent step: each case step whose check was just recorded ends at the flow as it is now."""
+        self.recorded = checks
+        for c in checks:
+            key = check_key(c)
+            # A check recorded again (the same expected result, new text) moves to where it was recorded last.
+            if key and self.check_text.get(key) != c.get("text"):
+                self.check_pos[key] = len(self.steps)
+                self.check_text[key] = c.get("text")
+        for c in checks:
+            n = c.get("step")
+            if isinstance(n, int) and 0 < n <= len(self.case_keys) and n not in self.check_at:
+                self.check_at[n] = len(self.steps)
+
+    def case_ends(self) -> List[Optional[int]]:
+        ends: List[Optional[int]] = [None] * len(self.case_keys)
+        for i, e in enumerate(self.carried_ends[: len(ends)]):
+            ends[i] = e
+        for n, at in self.check_at.items():
+            if ends[n - 1] is None:
+                ends[n - 1] = at
+        return ends
 
     def checkpoint(self) -> Optional[Dict[str, Any]]:
         """Save the steps that worked so far and the page memory learned so far, so a run cut off here can be picked up."""
@@ -447,7 +542,14 @@ class MemoryRun:
         if not folder:
             return None
         mode = str((req.memory or {}).get("replay") or "verify").lower()
-        return cls(MemoryStore(folder), mode if mode in REPLAY_MODES else "verify", req.goal, start_url)
+        mem = req.memory or {}
+        return cls(
+            MemoryStore(folder), mode if mode in REPLAY_MODES else "verify", req.goal, start_url,
+            case_id=req.case_id, case_keys=list(req.case_step_keys) if req.case_id else [],
+            checks_first=bool(mem.get("checks_first")),
+            canary_every=_int(mem.get("canary_every"), CANARY_EVERY),
+            max_age_days=_int(mem.get("max_age_days"), MAX_FLOW_AGE_DAYS),
+        )
 
     def finish(self, out: Dict[str, Any]) -> Dict[str, Any]:
         self.finished = True
@@ -470,6 +572,8 @@ class MemoryRun:
                 replayed=len(self.replay.done) if self.replay else 0,
                 llm_steps=self.llm_steps,
                 resume=self.resume if status == "fail" else None,
+                case={"id": self.case_id, "keys": self.case_keys, "ends": self.case_ends()} if self.case_id and self.case_keys else None,
+                checks=self.checks_to_save(status),
             )
         except Exception as ex:
             logger.warning("memory: could not save the flow: %s", ex)
@@ -494,6 +598,11 @@ class MemoryRun:
                 info["repaired"] = {"step": self.replay.stopped_at, "reason": self.replay.reason, "agent_steps": self.llm_steps}
         if self.replay and self.replay.heals:
             info["heals"] = self.replay.heals
+        if self.replay and self.replay.checked:
+            info["checked"] = len(self.replay.checked)
+            info["checks_held"] = sum(1 for c in self.replay.checked if c.get("holds_now"))
+        if self.replay and self.replay.skipped_reads:
+            info["skipped_reads"] = self.replay.skipped_reads
         if self.resume and status == "fail":
             info["resume"] = {
                 "steps": len(self.resume["steps"]),
@@ -788,7 +897,7 @@ async def run_browser_task(req: RunRequest, emit: Emit, cancel_event: asyncio.Ev
     except Exception as e:
         return {"success": False, "error": str(e), "summary": ""}
     llm_calls: List[Tuple[float, float]] = []
-    _time_llm_calls(llm, llm_calls)
+    _time_llm_calls(llm, llm_calls, lambda c: emit({"type": "call", **c}))
     use_prompts(req.prompts)
     diag = Diagnostics(accessibility=req.accessibility, network=req.network, axe=req.axe or None, vitals=req.vitals, visual=req.visual)
 
@@ -920,6 +1029,13 @@ async def run_browser_task(req: RunRequest, emit: Emit, cancel_event: asyncio.Ev
                 mem.used_checkpoint = str(checkpoint.get("run_id") or "")
             elif saved and isinstance(saved.get("resume"), dict) and saved["resume"].get("steps"):
                 flow, kind = saved["resume"], "resume"
+            elif mem.case_id and mem.case_keys:
+                # The test case was edited (its goal is new): replay its steps before the first one that changed.
+                earlier = mem.store.load_case_flow(mem.case_id)
+                if earlier and str(earlier.get("start_url") or "") == (normalized_initial or ""):
+                    done, end = MemoryStore.case_prefix(earlier, mem.case_keys)
+                    if done:
+                        flow, kind = {**earlier, "steps": earlier["steps"][:end], "case_steps_done": done}, "prefix"
         if flow and kind and flow.get("steps"):
             if kind == "interrupted":
                 replay_text = (
@@ -932,6 +1048,11 @@ async def run_browser_task(req: RunRequest, emit: Emit, cancel_event: asyncio.Ev
                     + (f" ({flow['run_id']})" if flow.get("run_id") else "")
                     + (f", which went wrong at step {flow['failed_at']}" if flow.get("failed_at") else "")
                 )
+            elif kind == "prefix":
+                replay_text = (
+                    f"The test case changed after step {flow['case_steps_done']}: replaying the {len(flow['steps'])} recorded "
+                    f"steps that did steps 1 to {flow['case_steps_done']}, which did not change"
+                )
             else:
                 replay_text = (
                     f"Replaying {len(flow['steps'])} recorded steps from memory "
@@ -940,13 +1061,19 @@ async def run_browser_task(req: RunRequest, emit: Emit, cancel_event: asyncio.Ev
             emit({"type": "memory", "text": replay_text})
             await browser.start()
             await diag.start(browser)
+            mem.flow_checks = [c for c in (flow.get("checks") or []) if isinstance(c, dict)] if mem.checks_first else []
             replay = await replay_flow(
-                flow, browser=browser, store=mem.store, emit=emit, cancel_event=cancel_event, on_step=lambda: diag.sample(browser), overlay=overlay
+                flow, browser=browser, store=mem.store, emit=emit, cancel_event=cancel_event, on_step=lambda: diag.sample(browser), overlay=overlay,
+                checks_first=mem.checks_first and kind == "exact",
             )
             mem.replay = replay
             for done_step in replay.done:
                 mem.add(done_step)
             replay_offset = replay.emitted
+            if replay.complete and kind in ("exact", "prefix"):
+                # The case steps the replay did end where they ended in the recorded run.
+                known = list(flow.get("case_step_ends") or [])
+                mem.carried_ends = known[: int(flow.get("case_steps_done") or len(known))]
             if replay.complete and kind == "exact":
                 emit({"type": "memory", "text": f"Replayed all {len(replay.done)} steps without the LLM"})
             elif replay.complete:
@@ -958,6 +1085,30 @@ async def run_browser_task(req: RunRequest, emit: Emit, cancel_event: asyncio.Ev
                 out = {"success": False, "error": err, "summary": err, "step_events": collected_steps, "cancelled": True}
                 _save(out, "cancelled")
                 return out
+            judged = judged_by_checks(flow, replay, mem) if replay.complete and kind == "exact" and mem.checks_first and mem.mode == "verify" else None
+            if judged and judged["ok"]:
+                held = replay.checked
+                summary = f"Replayed all {len(replay.done)} recorded steps and found all {len(held)} recorded checks on the page again, without calling the LLM."
+                if flow.get("summary"):
+                    summary += f"\n\nResult of the run that recorded them: {flow['summary']}"
+                emit({"type": "memory", "text": f"All {len(held)} recorded checks hold: passed without the LLM"})
+                out = {
+                    "success": True,
+                    "outcome": "passed",
+                    "summary": summary,
+                    "result_markdown": f"### Task\n\n{goal}\n\n---\n\n## Result\n\n{summary}",
+                    "session_id": req.session_id,
+                    "final_url": replay.url,
+                    "usage": {"input": 0, "output": 0},
+                    "steps": replay.emitted,
+                    "step_events": collected_steps,
+                    "checks": [{k: v for k, v in c.items() if k in ("expected", "text", "url", "step")} for c in held],
+                    "browser": await browser_profile(browser, browser_kw, provider=browser_provider, remote=browser_remote),
+                }
+                _save(out, "memory_checks")
+                return out
+            if judged and not judged["ok"]:
+                emit({"type": "memory", "text": f"The agent checks the result: {judged['why']}"})
             if replay.complete and mem.mode == "trust" and kind == "exact":
                 summary = f"Replayed all {len(replay.done)} recorded steps from memory without calling the LLM."
                 if flow.get("summary"):
@@ -1141,6 +1292,7 @@ async def run_browser_task(req: RunRequest, emit: Emit, cancel_event: asyncio.Ev
                     mem, pending_actions, results, mem.llm_steps, agent_note(getattr(last, "model_output", None)),
                     effects=list(action_effects), page_url=step_url["url"],
                 )
+                mem.note_checks(recorded_checks)
                 try:
                     point = mem.checkpoint()
                     if point:

package/src/agents.ts

@@ -0,0 +1,52 @@
+import type { SkillAgent } from "./skills/registry.ts"
+
+/**
+ * Every agent ForgeQE runs, what it loads on every model call, and how much that may be.
+ *
+ * This is the one list to change when an agent, a prompt, or a skill audience is added: tests/agents.test.ts checks it
+ * against the code (every prompt file is loaded by an agent, every agent the code defines is here) and checks each
+ * agent's fixed context against its budget, so the cost of a change shows in the build and in `forgeqe context`.
+ */
+export type AgentEntry = {
+  /** The agent runtime's name for it (AgentSpec name), or "engine" for the browser agent. */
+  name: string
+  title: string
+  /** agent-runtime: a session with tools; engine: the Python browser agent. */
+  kind: "agent-runtime" | "engine"
+  /** The prompt files it loads (prompts/<name>.md); the fragments they include count too. */
+  prompts: string[]
+  /** Skills it may load (by their `agents` audience); loaded on demand, so they are not in the fixed context. */
+  skills?: SkillAgent
+  /** Where the code defines it. */
+  defined: string
+  /** Most tokens its fixed context (system prompt, and for ForgeQE's agent its tool list) may take. */
+  budget: number
+}
+
+export const AGENTS: AgentEntry[] = [
+  { name: "forgeqe", title: "ForgeQE's agent (shell, forgeqe run, forgeqe mcp)", kind: "agent-runtime", prompts: ["brain"], skills: "forgeqe", defined: "src/brain/index.ts", budget: 26000 },
+  { name: "forgeqe-test", title: "Scribe: writes a test from a verified run", kind: "agent-runtime", prompts: ["test-agent"], skills: "scribe", defined: "src/code/runtime.ts", budget: 4900 },
+  { name: "forgeqe-read", title: "Scribe, reading the repository before the run arrives", kind: "agent-runtime", prompts: ["test-agent"], skills: "scribe", defined: "src/code/runtime.ts", budget: 4900 },
+  { name: "forgeqe-review", title: "Reviewer of Scribe's tests", kind: "agent-runtime", prompts: ["test-reviewer"], skills: "scribe", defined: "src/code/review.ts", budget: 4900 },
+  { name: "forgeqe-heal", title: "Scribe healing tests the application outgrew", kind: "agent-runtime", prompts: ["heal"], skills: "scribe", defined: "src/heal/loop.ts", budget: 4200 },
+  { name: "forgeqe-find", title: "Finds the repository tests a request means", kind: "agent-runtime", prompts: ["find-tests"], skills: "scribe", defined: "src/code/find.ts", budget: 1700 },
+  { name: "forgeqe-pr-review", title: "Reviews pull requests", kind: "agent-runtime", prompts: ["pr-review"], skills: "scribe", defined: "src/code/prreview.ts", budget: 1700 },
+  { name: "forgeqe-api", title: "Writes API and contract tests", kind: "agent-runtime", prompts: ["api-tests"], skills: "scribe", defined: "src/api/write.ts", budget: 2450 },
+  { name: "forgeqe-api-plan", title: "Plans end-to-end API scenarios", kind: "agent-runtime", prompts: ["api-scenarios"], defined: "src/api/plan.ts", budget: 1600 },
+  { name: "forgeqe-explore-plan", title: "Plans exploratory charters", kind: "agent-runtime", prompts: ["explore-charters"], defined: "src/explore/charters.ts", budget: 1650 },
+  { name: "forgeqe-defect-rca", title: "Finds the root cause of a defect", kind: "agent-runtime", prompts: ["defect-rca"], defined: "src/cases/rca.ts", budget: 950 },
+  { name: "forgeqe-refine", title: "Refine: turns a request into a task specification", kind: "agent-runtime", prompts: ["refine"], defined: "src/refine.ts", budget: 1450 },
+  { name: "forgeqe-judge", title: "Judges domain evaluations", kind: "agent-runtime", prompts: ["eval-judge"], defined: "src/evals/domain.ts", budget: 750 },
+  {
+    name: "engine",
+    title: "The browser agent (every browser run and test case)",
+    kind: "engine",
+    prompts: ["browser-api", "browser-capture", "browser-case-report", "browser-checks", "browser-efficiency", "browser-error-recovery", "browser-extra", "browser-goal-verification", "browser-skill-select", "conventions"],
+    skills: "browser",
+    defined: "py/forgeqe_engine/service.py",
+    budget: 2700,
+  },
+]
+
+/** Prompt files that are documentation, not loaded by any agent. */
+export const NOT_LOADED = ["README"]

package/src/brain/index.ts

@@ -8,6 +8,7 @@ import type { AppConfig } from "../config.ts"
 import { externalMcp, externalNotes, type ExternalPlan } from "../mcp/external.ts"
 import { MCP_NAME, TOOL_TIMEOUT_MS, localToolServer } from "../mcp/server.ts"
 import { TOOLS, environmentNotes, type ToolContext } from "../mcp/tools.ts"
+import { createRouter } from "../mcp/routing.ts"
 import { brainPrompt, BRAIN_SCHEMA, cleanFollowUps, type BrainReport, type FollowUp } from "./prompt.ts"
 import { redactSecrets } from "../sessionLogins.ts"
 import { logEvent } from "../sessionLog.ts"
@@ -115,11 +116,16 @@ export function carryOver(turns: { user: string; reply: string }[], text: string
   ].join("\n")
 }
 
+/** Routing (src/mcp/routing.ts) is on unless FORGEQE_ROUTING=off, kept for comparing and for finding a routing fault. */
+export const routingOn = (env: Record<string, string | undefined> = process.env) => !/^(off|0|false|no)$/i.test(env.FORGEQE_ROUTING || "")
+
 export async function openBrain(opts: BrainOptions): Promise<Brain> {
   const { config } = opts
   const cwd = realpathOr(opts.cwd)
   const context = (): ToolContext => ({ ...opts.context(), config, cwd })
-  const tools = localToolServer({ tools: TOOLS, context, instructions: () => environmentNotes(context()), timeoutMs: TOOL_TIMEOUT_MS })
+  // Tools listed in one line and playbooks on demand keep every model call small; FORGEQE_ROUTING=off sends it all.
+  const router = routingOn() ? createRouter(TOOLS) : undefined
+  const tools = localToolServer({ tools: TOOLS, context, router, instructions: () => environmentNotes(context()), timeoutMs: TOOL_TIMEOUT_MS })
   const external = opts.external || externalMcp(config, Bun.which, process.env, { cwd })
   const notes = [environmentNotes(context()), ...externalNotes(external), ...(opts.notes || [])].filter(Boolean).join("\n")
   const readDirs = [...new Set([config.export.outDir, config.memory.dir, cwd].flatMap((d) => [path.resolve(d), realpathOr(d)]))]
@@ -147,7 +153,7 @@ export async function openBrain(opts: BrainOptions): Promise<Brain> {
       agent: {
         name: BRAIN_AGENT,
         description: "ForgeQE: browser runs, test cases, tests, and CI pipelines",
-        prompt: `${brainPrompt()}\n\nThis session:\n${notes}`,
+        prompt: `${router ? router.parts.core : brainPrompt()}\n\nThis session:\n${notes}`,
         skills: "forgeqe",
         // Systems are read through MCP or query_api, never by scripts in the shell that handle tokens themselves.
         permission: { bash: agentBashPermission(privateEnvNames(config)) },