@capagents/forgeqe 1.2.0 → 1.3.0

package/py/forgeqe_engine/llm.py

@@ -6,7 +6,7 @@ import logging
 import os
 import re
 import time
-from typing import Any, Dict, List, Optional, Tuple
+from typing import Any, Callable, Dict, List, Optional, Tuple
 
 logger = logging.getLogger("forgeqe_engine.service")
 
@@ -157,18 +157,46 @@ def _build_llm(llm_config: Optional[Dict[str, Any]]):
     raise ValueError(f"Unsupported LLM provider for the engine: {provider}")
 
 
-def _time_llm_calls(llm: Any, calls: List[Tuple[float, float]]) -> None:
-    """Record when each model call started and ended, so the report can split step time into model and browser."""
+def call_usage(result: Any) -> Dict[str, int]:
+    """One model call's tokens from its completion (browser-use ChatInvokeUsage); prompt tokens include cached ones."""
+    u = getattr(result, "usage", None)
+    if u is None:
+        return {}
+    out = {"input": int(getattr(u, "prompt_tokens", 0) or 0), "output": int(getattr(u, "completion_tokens", 0) or 0)}
+    cached = int(getattr(u, "prompt_cached_tokens", 0) or 0)
+    written = int(getattr(u, "prompt_cache_creation_tokens", 0) or 0)
+    if cached:
+        out["cached"] = cached
+    if written:
+        out["cache_write"] = written
+    return out
+
+
+def _time_llm_calls(llm: Any, calls: List[Tuple[float, float]], on_call: Optional[Callable[[Dict[str, Any]], None]] = None) -> None:
+    """
+    Record when each model call started and ended, so the report can split step time into model and browser;
+    on_call also gets each call's time and tokens (counts only, never the prompt), for ForgeQE's usage ledger.
+    """
     inner = getattr(llm, "ainvoke", None)
     if inner is None:
         return
 
     async def timed(*args: Any, **kwargs: Any) -> Any:
         started = time.time()
+        result: Any = None
+        ok = False
         try:
-            return await inner(*args, **kwargs)
+            result = await inner(*args, **kwargs)
+            ok = True
+            return result
         finally:
-            calls.append((started, time.time()))
+            ended = time.time()
+            calls.append((started, ended))
+            if on_call is not None:
+                try:
+                    on_call({"ms": int((ended - started) * 1000), "ok": ok, **call_usage(result)})
+                except Exception:
+                    logger.debug("model call usage not reported", exc_info=True)
 
     try:
         object.__setattr__(llm, "ainvoke", timed)

package/py/forgeqe_engine/memory.py

@@ -5,6 +5,7 @@ Layout under the memory directory:
 
   pages/<host>/<page>.json   every interactive element seen on that page, each with ranked locators
   flows/<goal>-<hash>.json   the actions of the last successful run of a goal, replayable without the LLM
+  cases/<hash>.json          for a test case: which flow it last passed with, so an edited case replays its unchanged steps
 
 A locator's confidence (1-99 %) multiplies four factors: how stable its kind is (test id > id >
 role+name > ... > xpath), how unique it was on the page, how consistently it reappeared on later
@@ -650,22 +651,44 @@ class MemoryStore:
         replayed: int = 0,
         llm_steps: int = 0,
         resume: Optional[Dict[str, Any]] = None,
+        case: Optional[Dict[str, Any]] = None,
+        checks: Optional[List[Dict[str, Any]]] = None,
     ) -> Optional[str]:
         """
         Update the goal's flow after a run. Only successful runs replace its steps. Returns "saved" or "updated".
 
         A failed run with `resume` (the steps that worked before it went wrong) keeps them under "resume", so the
         next run of this task replays them and the agent picks up where the failed run went wrong.
+
+        `case` ({id, keys, ends}) is the test case the goal came from: a key per case step, and for each the number
+        of flow steps done by the end of it (None where unknown). A successful run records it and points the case's
+        index at this flow, so a run of the case after an edit replays the steps before the first one that changed.
         """
         if status == "stopped":
             return None
         with _locked(self.root):
             return self._save_flow_result(goal, start_url, status=status, steps=steps, summary=summary,
-                                          final_url=final_url, replayed=replayed, llm_steps=llm_steps, resume=resume)
+                                          final_url=final_url, replayed=replayed, llm_steps=llm_steps, resume=resume, case=case,
+                                          checks=checks)
+
+    def case_file(self, case_id: str) -> Path:
+        return self.root / "cases" / f"{_sha(case_id, 16)}.json"
+
+    def load_case_flow(self, case_id: str) -> Optional[Dict[str, Any]]:
+        """The flow a test case last passed with, when it recorded where each case step ended."""
+        index = _read_json(self.case_file(case_id)) if case_id else None
+        name = str((index or {}).get("flow") or "")
+        if not name or "/" in name or "\\" in name:
+            return None
+        flow = _read_json(self.root / "flows" / name)
+        if not flow or flow.get("case_id") != case_id or not flow.get("successes") or not isinstance(flow.get("steps"), list):
+            return None
+        return flow
 
     def _save_flow_result(self, goal: str, start_url: Optional[str], *, status: str, steps: List[Dict[str, Any]],
                           summary: str, final_url: str, replayed: int, llm_steps: int,
-                          resume: Optional[Dict[str, Any]] = None) -> Optional[str]:
+                          resume: Optional[Dict[str, Any]] = None, case: Optional[Dict[str, Any]] = None,
+                          checks: Optional[List[Dict[str, Any]]] = None) -> Optional[str]:
         file = self.flow_file(goal, start_url)
         flow = self.load_flow(goal, start_url)
         created = flow is None
@@ -689,6 +712,18 @@ class MemoryStore:
             flow["llm_steps"] = llm_steps
             flow["replayed_steps"] = replayed
             flow.pop("resume", None)
+            # checks: what the run checked, with the flow step each came after (observe.py); None keeps the flow's.
+            # New steps from the agent with no checks given leave none, since old positions would not fit them.
+            if checks is not None:
+                flow["checks"] = checks
+            elif llm_steps and flow.get("checks"):
+                flow.pop("checks", None)
+            if case and case.get("id") and case.get("keys"):
+                kept = len(flow["steps"])
+                flow["case_id"] = str(case["id"])
+                flow["case_step_keys"] = [str(k) for k in case["keys"]]
+                flow["case_step_ends"] = [e if isinstance(e, int) and 0 < e <= kept else None for e in (case.get("ends") or [])][: len(case["keys"])]
+                _write_json(self.case_file(str(case["id"])), {"case_id": str(case["id"]), "flow": file.name, "updated": now})
         elif status != "complete":
             flow["failures"] = int(flow.get("failures") or 0) + 1
         if keep_resume and resume:
@@ -702,6 +737,28 @@ class MemoryStore:
             return None
         return "saved" if created else "updated"
 
+    @staticmethod
+    def case_prefix(flow: Dict[str, Any], keys: List[str]) -> Tuple[int, int]:
+        """
+        How much of a case's earlier flow still fits its steps now: (case steps, flow steps). The case steps are the
+        leading ones whose keys did not change, up to the last of them where the flow recorded its end. (0, 0) when none.
+
+        An end is trusted only when the next step that recorded one ended later: when the agent did the actions of two
+        steps before recording either check, both ends are the same, and replaying up to the first would also redo
+        the second step's actions, which may be the step that changed.
+        """
+        old = [str(k) for k in (flow.get("case_step_keys") or [])]
+        ends = [e if isinstance(e, int) and 0 < e <= len(flow.get("steps") or []) else None for e in (flow.get("case_step_ends") or [])]
+        same = 0
+        while same < min(len(old), len(keys)) and old[same] == keys[same]:
+            same += 1
+        for n in range(same, 0, -1):
+            end = ends[n - 1] if n - 1 < len(ends) else None
+            later = next((e for e in ends[n:] if e is not None), None)
+            if end is not None and (later is None or later > end):
+                return n, end
+        return 0, 0
+
     def recent_flows(self, host: str, *, exclude: Optional[Path] = None, limit: int = 2) -> List[Dict[str, Any]]:
         folder = self.root / "flows"
         if not folder.is_dir():
@@ -1001,4 +1058,15 @@ def _compact_step(step: Dict[str, Any]) -> Dict[str, Any]:
         return {"op": "back"}
     if action == "evaluate":
         return {"op": "evaluate", "code": str(p.get("code") or "")[:2000]}
+    if action == "api_request":
+        # An HTTP call made in the page's session: what was sent, and what the server answered when the run made it.
+        out = {"op": "api", "method": str(p.get("method") or "GET").upper(), "url": p.get("url") or ""}
+        for k in ("body", "content_type", "csrf"):
+            if p.get(k):
+                out[k] = p[k]
+        answer = step.get("api") or {}
+        for k in ("status", "final_url", "redirected", "snippet"):
+            if answer.get(k) not in (None, "", False):
+                out[k] = answer[k]
+        return out
     return {"op": "unknown", "action_keys": [action]}

package/py/forgeqe_engine/observe.py

@@ -0,0 +1,123 @@
+"""
+Replaying a flow without the model, and knowing it still passes (docs/COST_ARCHITECTURE.md, phase 2).
+
+Two parts:
+- read_only_script: whether an `evaluate` step in a flow only read the page. The agent often runs a script to look at
+  the page (list the fields, read their values) before it decides; replay can leave such a step out. The test is on
+  the script's source and conservative: anything that could write (assign, call a method that changes the page or the
+  browser's state, send a request, schedule code) counts as writing, and the step then needs the agent as before.
+- checks_for_flow / flow_checks_hold: the results a passing run recorded (record_check), kept in its flow with the
+  step each was recorded after, and checked again on the replayed page with no model.
+"""
+from __future__ import annotations
+
+import asyncio
+import re
+from typing import Any, Awaitable, Callable, Dict, List, Optional
+
+# Anything that writes to the page, the browser, or the network, or runs code later. A script with any of these
+# is not read-only. False positives only cost a model call (today's path); a false negative could skip a real step,
+# so the list leans wide.
+_WRITES = re.compile(
+    r"""
+    (?<![=!<>])=(?![=>])                                   # any assignment (=, +=, -=, ...), not ==, ===, =>, <=, >=, !=
+    | \+\+ | --                                            # increments
+    | \b(?:delete)\b
+    | \.(?:click|submit|requestSubmit|reset|focus|blur|select|setSelectionRange|setRangeText|showModal|show|close|open|
+         dispatchEvent|remove|removeChild|append|appendChild|prepend|insertBefore|insertAdjacent\w*|replaceChild|
+         replaceWith|replaceChildren|before|after|setAttribute\w*|removeAttribute\w*|toggleAttribute|
+         scroll\w*|play|pause|load|reload|assign|pushState|replaceState|back|forward|go|
+         setItem|removeItem|clear|add|toggle|set|setProperty|removeProperty|attachShadow|
+         sendBeacon|postMessage|writeText|write|writeln|execCommand|requestFullscreen|exitFullscreen)\s*\(
+    | \b(?:fetch|XMLHttpRequest|WebSocket|EventSource|eval|Function|setTimeout|setInterval|requestAnimationFrame|
+         queueMicrotask|importScripts|import)\s*\(
+    | \bnew\s+(?:\w*Event|XMLHttpRequest|WebSocket|Worker|SharedWorker|Function|MutationObserver|BroadcastChannel)\b
+    | \b(?:document\.cookie|location|indexedDB|caches|navigator\.serviceWorker)\b
+    """,
+    re.VERBOSE,
+)
+
+# Declarations (const x = ..., let x = ..., var x = ..., function parameters with defaults) write only local names.
+_DECLARATION = re.compile(r"\b(?:const|let|var)\s+(?:[A-Za-z_$][\w$]*|\{[^}]*\}|\[[^\]]*\])\s*=(?![=>])")
+# Text inside string and template literals: an "=" or ".click(" in a selector or a message is not code.
+_STRINGS = re.compile(r"""'(?:\\.|[^'\\])*'|"(?:\\.|[^"\\])*"|`(?:\\.|[^`\\])*`""", re.DOTALL)
+# Comments.
+_COMMENTS = re.compile(r"//[^\n]*|/\*.*?\*/", re.DOTALL)
+
+
+def read_only_script(code: str) -> bool:
+    """Whether a script only reads the page: no assignment to anything but its own new local names, no call that changes
+    the page, the browser, or storage, no request, and no code run later."""
+    src = str(code or "")
+    if not src.strip() or len(src) > 20_000:
+        return False
+    src = _COMMENTS.sub(" ", src)
+    src = _STRINGS.sub('""', src)
+    # `const x = ...`: the assignment is the declaration of a local name. Remove the "=" of each declaration only;
+    # the expression after it is still checked.
+    src = _DECLARATION.sub(lambda m: m.group(0)[:-1] + " ", src)
+    # Parameters with defaults, `(a = 1) =>`: rare in these scripts; treated as an assignment, so not read-only.
+    return _WRITES.search(src) is None
+
+
+def read_only_step(step: Dict[str, Any]) -> bool:
+    """A flow step replay can leave out: an `evaluate` whose script only reads the page."""
+    return step.get("action") == "evaluate" and read_only_script(str((step.get("params") or {}).get("code") or ""))
+
+
+# ---------------------------------------------------------------------------------------------
+# Recorded checks
+# ---------------------------------------------------------------------------------------------
+
+CHECK_ATTEMPTS = 6
+CHECK_DELAY_SECONDS = 0.5
+
+
+def checks_for_flow(checks: List[Dict[str, Any]], at: Dict[str, int]) -> List[Dict[str, Any]]:
+    """The checks a passing run recorded, as its flow keeps them: what to find on the page, and after which flow step."""
+    out: List[Dict[str, Any]] = []
+    for c in checks or []:
+        text = " ".join(str(c.get("text") or "").split())
+        if not text:
+            continue
+        entry: Dict[str, Any] = {"expected": str(c.get("expected") or text)[:200], "text": text[:200], "at": int(at.get(check_key(c), 0))}
+        if c.get("url"):
+            entry["url"] = str(c["url"])
+        if c.get("dynamic"):
+            entry["dynamic"] = True
+        if c.get("holds") is False:
+            entry["holds"] = False
+        if isinstance(c.get("step"), int) and c["step"] > 0:
+            entry["step"] = c["step"]
+        out.append(entry)
+    return out
+
+
+def check_key(c: Dict[str, Any]) -> str:
+    return str(c.get("expected") or c.get("text") or "").strip().lower()
+
+
+def checkable(flow: Dict[str, Any]) -> Optional[str]:
+    """None when the flow's checks can decide a replay on their own; otherwise why not, in words."""
+    checks = flow.get("checks")
+    if not isinstance(checks, list) or not checks:
+        return "the flow has no recorded checks"
+    if any(c.get("dynamic") for c in checks):
+        return "a recorded check is live data (dynamic), which only the agent can judge"
+    if any(c.get("holds") is False for c in checks):
+        return "a recorded check did not hold when it was recorded"
+    return None
+
+
+async def check_holds(shows: Callable[[str], Awaitable[Optional[bool]]], text: str, attempts: Optional[int] = None, delay: Optional[float] = None) -> Optional[bool]:
+    """Whether the page shows `text`, asking again for a few seconds while it settles; None when it could not be asked."""
+    attempts = CHECK_ATTEMPTS if attempts is None else attempts
+    delay = CHECK_DELAY_SECONDS if delay is None else delay
+    seen: Optional[bool] = None
+    for n in range(max(1, attempts)):
+        seen = await shows(text)
+        if seen:
+            return True
+        if n + 1 < attempts:
+            await asyncio.sleep(delay)
+    return seen

package/py/forgeqe_engine/replay.py

@@ -16,6 +16,7 @@ from typing import Any, Awaitable, Callable, Dict, List, Optional, Tuple
 
 from . import brand
 from .logins import totp_code
+from .observe import check_holds, read_only_step
 from .memory import (
     ELEMENT_ACTIONS,
     READ_ONLY_ACTIONS,
@@ -45,6 +46,10 @@ class ReplayOutcome:
     times: List[Tuple[float, float]] = field(default_factory=list)
     # Steps whose recorded locator no longer matched and another one found the element.
     heals: List[Dict[str, Any]] = field(default_factory=list)
+    # The flow's recorded checks found on the page again, each after the step it was recorded after (observe.py).
+    checked: List[Dict[str, Any]] = field(default_factory=list)
+    # Steps left out because they only read the page (an `evaluate` with a read-only script).
+    skipped_reads: int = 0
 
     @property
     def complete(self) -> bool:
@@ -86,7 +91,13 @@ async def replay_flow(
     cancel_event: asyncio.Event,
     on_step: Optional[Callable[[], Awaitable[None]]] = None,
     overlay: Any = None,
+    checks_first: bool = False,
 ) -> ReplayOutcome:
+    """
+    `checks_first` (registry.mode on, docs/COST_ARCHITECTURE.md): steps that only read the page are left out, and the
+    flow's recorded checks are found on the page again after the step each was recorded after. A check that does not
+    hold stops the replay there, so the agent takes over on that page as after any other stop.
+    """
     from browser_use import Tools
 
     tools = Tools()
@@ -99,6 +110,23 @@ async def replay_flow(
         out.reason = reason
         return out
 
+    checks = [c for c in (flow.get("checks") or []) if isinstance(c, dict) and c.get("text")] if checks_first else []
+
+    async def verify_checks(after: int) -> Optional[str]:
+        """Finds again the checks recorded after flow step `after`; the first that does not hold, in words."""
+        due = [c for c in checks if (int(c.get("at") or 0) == after) or (after == len(steps) and not 0 < int(c.get("at") or 0) <= len(steps))]
+        if not due:
+            return None
+        from .checks import page_shows
+
+        for c in due:
+            holds = await check_holds(lambda t: page_shows(browser, t), str(c["text"]))
+            out.checked.append({**c, "holds_now": bool(holds)})
+            emit({"type": "result", "step": after, "items": [{"ok": bool(holds), "done": False, "text": f'check "{c.get("expected") or c["text"]}": {"shows" if holds else "does not show"} "{c["text"]}"'[:240]}]})
+            if not holds:
+                return f'the recorded check "{c.get("expected") or c["text"]}" does not hold: the page does not show "{c["text"]}"'
+        return None
+
     for n, step in enumerate(steps, 1):
         if cancel_event.is_set():
             stop(n, "stopped by user")
@@ -106,6 +134,17 @@ async def replay_flow(
         action = str(step.get("action") or "")
         params = dict(step.get("params") or {})
         line = str(step.get("line") or action)
+        if checks_first and read_only_step(step):
+            # The agent looked at the page here; replay has nothing to do. Kept in the flow so check positions hold.
+            out.done.append(step)
+            out.times.append((time.time(), time.time()))
+            out.emitted = n
+            out.skipped_reads += 1
+            failed = await verify_checks(n)
+            if failed:
+                stop(n + 1 if n < len(steps) else n, failed)
+                break
+            continue
         if action not in REPLAYABLE_ACTIONS:
             stop(n, f"`{line}` needs the agent")
             break
@@ -188,6 +227,10 @@ async def replay_flow(
                 out.heals.append({"step": n, "line": line, "label": step.get("label") or "", "stale": [_locator_text(k) for k in gone], "used": _locator_text(used)})
         if on_step is not None:
             await on_step()
+        failed = await verify_checks(n)
+        if failed:
+            stop(n + 1 if n < len(steps) else n, failed)
+            break
 
     out.url = await _current_url(browser)
     return out
@@ -197,11 +240,14 @@ def handoff_task(goal: str, flow: Dict[str, Any], outcome: ReplayOutcome, *, kin
     """
     The agent's task after a replay: the goal, what already happened, and where to pick up.
 
-    `kind` is "exact" (an earlier successful run of this task), "resume" (the steps that worked in a run of this
+    `kind` is "exact" (an earlier successful run of this task), "prefix" (the recorded steps of a test case's unchanged
+    first steps, after the case was edited), "resume" (the steps that worked in a run of this
     task that later went wrong), or "interrupted" (the steps a run of this task got through before it was cut off: closed, crashed, or the machine restarted).
     """
     when = str(flow.get("last_success") or flow.get("at") or "")[:10]
-    if kind == "resume":
+    if kind == "prefix":
+        source = "an earlier successful run of this test case, before its later steps were changed"
+    elif kind == "resume":
         source = "an earlier run of this task that went wrong later"
     elif kind == "interrupted":
         source = "an earlier run of this task that was cut off before it finished"
@@ -221,6 +267,13 @@ def handoff_task(goal: str, flow: Dict[str, Any], outcome: ReplayOutcome, *, kin
             f"All recorded steps ran and the browser is on {outcome.url or 'the final page'}. Check this page: "
             "if the task is complete, call done right away and report the result. Otherwise continue from here."
         )
+    elif kind == "prefix":
+        n = int(flow.get("case_steps_done") or 0)
+        lines.append(
+            f"They did test steps 1 to {n}, which have not changed. The browser is on {outcome.url or 'the current page'}. "
+            f"Check that the page agrees, then continue with test step {n + 1}. In your step report, report steps 1 to {n} "
+            "from what the page shows now."
+        )
     elif kind == "interrupted":
         lines.append(
             f"That is as far as the earlier run got before it was cut off. The browser is on {outcome.url or 'the current page'}. "