@capagents/forgeqe 0.0.0-stage → 1.0.0

package/py/forgeqe_engine/network.py

@@ -0,0 +1,244 @@
+"""
+The application's API calls during a run (XHR, fetch, event streams), for API and contract tests: method, URL, status,
+timing, and the request and response bodies when they are JSON, XML or text and small. Secrets never reach the file:
+credentials in headers are dropped (only the kind of auth is kept), and fields named like a password, token or key are
+redacted in bodies and query strings.
+"""
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+import re
+import time
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
+
+logger = logging.getLogger(__name__)
+
+MAX_CALLS = 300
+MAX_BODY = 64_000
+API_TYPES = {"XHR", "Fetch", "EventSource"}
+TEXT_MIME = re.compile(r"json|xml|text/|graphql|x-www-form-urlencoded|grpc-web-text", re.I)
+SECRET_NAME = re.compile(
+    r"pass(word|wd|phrase)?$|secret|token|api[-_]?key|auth|cookie|session|^sid$|otp|^pin$|cvv|cvc|card.?number|^pan$|ssn|private[-_]?key|signature",
+    re.I,
+)
+KEEP_HEADERS = {"content-type", "accept", "soapaction", "x-requested-with", "grpc-status", "grpc-message", "x-grpc-web", "x-apollo-operation-name"}
+TRACKERS = re.compile(
+    r"google-analytics|googletagmanager|doubleclick|googlesyndication|facebook\.(com|net)|segment\.(io|com)|hotjar|sentry\.io|"
+    r"newrelic|nr-data|datadoghq|clarity\.ms|mixpanel|amplitude|intercom|fullstory|launchdarkly|optimizely|bugsnag|appdynamics|"
+    r"dynatrace|applicationinsights|visualstudio\.com/v2/track|browser-intake",
+    re.I,
+)
+REDACTED = "<redacted>"
+
+
+def redact_json(value: Any) -> Any:
+    if isinstance(value, dict):
+        return {k: (REDACTED if SECRET_NAME.search(str(k)) and value[k] not in (None, "") else redact_json(v)) for k, v in value.items()}
+    if isinstance(value, list):
+        return [redact_json(v) for v in value]
+    return value
+
+
+def redact_xml(text: str) -> str:
+    def element(m: "re.Match[str]") -> str:
+        return f"{m.group(1)}{REDACTED}{m.group(3)}" if SECRET_NAME.search(m.group(2)) else m.group(0)
+
+    return re.sub(r"(<(?:\w+:)?([\w.-]+)(?:\s[^>]*)?>)[^<]+(</(?:\w+:)?\2>)", element, text)
+
+
+def redact_query(url: str) -> str:
+    try:
+        u = urlsplit(url)
+    except ValueError:
+        return url
+    if not u.query:
+        return url
+    pairs = [(k, REDACTED if SECRET_NAME.search(k) and v else v) for k, v in parse_qsl(u.query, keep_blank_values=True)]
+    return urlunsplit((u.scheme, u.netloc, u.path, urlencode(pairs, safe="<>"), ""))
+
+
+def redact_body(text: str, mime: str) -> str:
+    """A body with its secret fields redacted: JSON by key, XML by element, a form by field, other text as is."""
+    t = text.strip()
+    if "json" in mime or (t[:1] in "{[" and t[-1:] in "}]"):
+        try:
+            return json.dumps(redact_json(json.loads(t)), ensure_ascii=False)
+        except ValueError:
+            pass
+    if "xml" in mime or t.startswith("<"):
+        return redact_xml(text)
+    if "x-www-form-urlencoded" in mime:
+        return urlencode([(k, REDACTED if SECRET_NAME.search(k) and v else v) for k, v in parse_qsl(text, keep_blank_values=True)], safe="<>")
+    return text
+
+
+def auth_kind(headers: Dict[str, Any]) -> Optional[Dict[str, str]]:
+    """How the request authenticated, without the credential: bearer, basic, cookie, or the API key header's name."""
+    low = {str(k).lower(): str(v) for k, v in (headers or {}).items()}
+    if "authorization" in low:
+        scheme = low["authorization"].split(" ", 1)[0].lower()
+        return {"type": scheme if scheme in ("bearer", "basic", "digest", "negotiate", "ntlm") else "authorization"}
+    for k in low:
+        if k != "cookie" and SECRET_NAME.search(k) and k not in KEEP_HEADERS:
+            return {"type": "header", "name": k}
+    if "cookie" in low:
+        return {"type": "cookie"}
+    return None
+
+
+def _site(host: str) -> str:
+    parts = host.split(":")[0].split(".")
+    return ".".join(parts[-2:]) if len(parts) >= 2 else host
+
+
+class NetworkCapture:
+    def __init__(self) -> None:
+        self.calls: List[Dict[str, Any]] = []
+        self._pending: Dict[str, Dict[str, Any]] = {}
+        self._tasks: List[asyncio.Task[Any]] = []
+        self.page_url = ""
+        self.dropped = 0
+
+    def wanted(self, params: Dict[str, Any]) -> bool:
+        url = str((params.get("request") or {}).get("url") or "")
+        return str(params.get("type") or "") in API_TYPES and url.startswith("http") and not TRACKERS.search(url)
+
+    def on_request(self, params: Any, session_id: Optional[str] = None) -> None:
+        try:
+            if not self.wanted(params):
+                if str(params.get("type") or "") == "Document":
+                    self.page_url = str((params.get("request") or {}).get("url") or self.page_url)
+                return
+            if len(self.calls) + len(self._pending) >= MAX_CALLS:
+                self.dropped += 1
+                return
+            req = params.get("request") or {}
+            headers = req.get("headers") or {}
+            mime = str(next((v for k, v in headers.items() if str(k).lower() == "content-type"), ""))
+            call: Dict[str, Any] = {
+                "t": float(params.get("wallTime") or time.time()),
+                "method": str(req.get("method") or "GET"),
+                "url": redact_query(str(req.get("url") or "")),
+                "type": str(params.get("type")),
+                "request": {"headers": {k: v for k, v in headers.items() if str(k).lower() in KEEP_HEADERS}},
+                "_ts": float(params.get("timestamp") or 0),
+                "_session": session_id,
+                "_has_post": bool(req.get("hasPostData")),
+            }
+            auth = auth_kind(headers)
+            if auth:
+                call["auth"] = auth
+            if self.page_url:
+                call["page"] = redact_query(self.page_url)
+                page_host, host = urlsplit(self.page_url).hostname or "", urlsplit(call["url"]).hostname or ""
+                if page_host and host and _site(page_host) != _site(host):
+                    call["third_party"] = True
+            post = req.get("postData")
+            if isinstance(post, str) and post:
+                call["request"]["body"] = redact_body(post[:MAX_BODY], mime)
+                if len(post) > MAX_BODY:
+                    call["request"]["truncated"] = True
+            self._pending[str(params.get("requestId"))] = call
+        except Exception:
+            logger.debug("request not recorded", exc_info=True)
+
+    def on_response(self, params: Any, session_id: Optional[str] = None) -> None:
+        try:
+            call = self._pending.get(str(params.get("requestId")))
+            if call is None:
+                return
+            res = params.get("response") or {}
+            headers = {k: v for k, v in (res.get("headers") or {}).items() if str(k).lower() in KEEP_HEADERS}
+            call["response"] = {"status": int(res.get("status") or 0), "mime": str(res.get("mimeType") or ""), "headers": headers}
+        except Exception:
+            logger.debug("response not recorded", exc_info=True)
+
+    def on_finished(self, params: Any, session_id: Optional[str] = None, client: Any = None) -> None:
+        call = self._pending.pop(str(params.get("requestId")), None)
+        if call is None:
+            return
+        if call["_ts"]:
+            call["ms"] = max(0, int((float(params.get("timestamp") or call["_ts"]) - call["_ts"]) * 1000))
+        call.setdefault("response", {"status": 0, "mime": "", "headers": {}})
+        call["response"]["size"] = int(params.get("encodedDataLength") or 0)
+        self.calls.append(call)
+        if client is not None:
+            # Bodies are fetched off the event loop: waiting for a reply inside an event handler would block the handler's own reader.
+            self._tasks.append(asyncio.ensure_future(self._bodies(client, str(params.get("requestId")), call)))
+
+    def on_failed(self, params: Any, session_id: Optional[str] = None) -> None:
+        call = self._pending.pop(str(params.get("requestId")), None)
+        if call is None or params.get("canceled"):
+            return
+        call["failed"] = str(params.get("errorText") or "failed")
+        call.setdefault("response", {"status": 0, "mime": "", "headers": {}})
+        self.calls.append(call)
+
+    async def _bodies(self, client: Any, request_id: str, call: Dict[str, Any]) -> None:
+        sid = call.get("_session")
+        try:
+            if call["_has_post"] and "body" not in call["request"]:
+                r = await client.send.Network.getRequestPostData(params={"requestId": request_id}, session_id=sid)
+                post = str((r or {}).get("postData") or "")
+                mime = str(call["request"]["headers"].get("Content-Type") or call["request"]["headers"].get("content-type") or "")
+                if post:
+                    call["request"]["body"] = redact_body(post[:MAX_BODY], mime)
+        except Exception:
+            logger.debug("request body not read", exc_info=True)
+        res = call["response"]
+        if not TEXT_MIME.search(res.get("mime") or "") or res.get("size", 0) > MAX_BODY * 4:
+            return
+        try:
+            r = await client.send.Network.getResponseBody(params={"requestId": request_id}, session_id=sid)
+            if (r or {}).get("base64Encoded"):
+                return
+            body = str((r or {}).get("body") or "")
+            res["body"] = redact_body(body[:MAX_BODY], res.get("mime") or "")
+            if len(body) > MAX_BODY:
+                res["truncated"] = True
+        except Exception:
+            logger.debug("response body not read", exc_info=True)
+
+    async def settle(self, timeout: float = 5.0) -> None:
+        pending = [t for t in self._tasks if not t.done()]
+        if pending:
+            await asyncio.wait(pending, timeout=timeout)
+
+    def write(self, out_dir: Path) -> Optional[str]:
+        if not self.calls:
+            return None
+        calls = [{k: v for k, v in c.items() if not k.startswith("_")} for c in sorted(self.calls, key=lambda c: c["t"])]
+        file = out_dir / "network.json"
+        file.write_text(json.dumps({"version": 1, "calls": calls, **({"dropped": self.dropped} if self.dropped else {})}, indent=1, ensure_ascii=False), encoding="utf-8")
+        return str(file.resolve())
+
+
+HOOKED = "_forgeqe_hook"
+
+
+def hook(client: Any, method: str, fn: Any) -> None:
+    """
+    Add a handler for a CDP event without replacing the one already there: the CDP client keeps one handler per event,
+    and the library's own watchdogs (downloads, HAR) listen to the same network events.
+    """
+    registry = getattr(client, "_event_registry", None)
+    handlers = getattr(registry, "_handlers", None)
+    if handlers is None:
+        return
+    current = handlers.get(method)
+    if current is not None and getattr(current, HOOKED, None) is fn:
+        return
+
+    def both(params: Any, session_id: Optional[str] = None) -> Any:
+        try:
+            fn(params, session_id)
+        except Exception:
+            logger.debug("network handler failed", exc_info=True)
+        return current(params, session_id) if current is not None else None
+
+    setattr(both, HOOKED, fn)
+    registry.register(method, both)

package/py/forgeqe_engine/prompts.py

@@ -0,0 +1,20 @@
+"""
+The browser agent's instructions, configurable: ForgeQE sends each prompts/browser-*.md (the project's edited copy, or
+its own) with the run request, and the engine uses those. The constants in the engine's modules are the same text, for
+the engine on its own (tests, `python -m forgeqe_engine`); tests/engine/test_prompts.py keeps the two in step.
+"""
+from __future__ import annotations
+
+from typing import Dict
+
+_OVERRIDES: Dict[str, str] = {}
+
+
+def use_prompts(prompts: Dict[str, str]) -> None:
+    """Set the prompts for this run (name -> text); a name not given keeps the engine's own."""
+    _OVERRIDES.clear()
+    _OVERRIDES.update({str(k): str(v) for k, v in (prompts or {}).items() if isinstance(v, str)})
+
+
+def text(name: str, default: str) -> str:
+    return _OVERRIDES.get(name, default)

package/py/forgeqe_engine/replay.py

@@ -0,0 +1,240 @@
+"""
+Replay a recorded flow inside the run's own library session, without calling the LLM.
+
+Each element step re-finds its element on the live page through the memory's ranked locators and
+runs the same library action with the element's new index. Replay stops at the first step it
+cannot do safely; the agent then continues from the page replay reached.
+"""
+from __future__ import annotations
+
+import asyncio
+import logging
+import os
+import time
+from dataclasses import dataclass, field
+from typing import Any, Awaitable, Callable, Dict, List, Optional, Tuple
+
+from . import brand
+from .logins import totp_code
+from .memory import (
+    ELEMENT_ACTIONS,
+    READ_ONLY_ACTIONS,
+    REPLAYABLE_ACTIONS,
+    MemoryStore,
+    flow_step,
+    resolve_step,
+)
+
+logger = logging.getLogger(__name__)
+
+FIND_ATTEMPTS = 6
+FIND_DELAY_SECONDS = 0.8
+
+Emit = Callable[[Dict[str, Any]], None]
+
+
+@dataclass
+class ReplayOutcome:
+    total: int
+    done: List[Dict[str, Any]] = field(default_factory=list)
+    emitted: int = 0
+    stopped_at: Optional[int] = None
+    reason: str = ""
+    url: str = ""
+    # Start and end time of each step in `done`, for the report's timeline.
+    times: List[Tuple[float, float]] = field(default_factory=list)
+    # Steps whose recorded locator no longer matched and another one found the element.
+    heals: List[Dict[str, Any]] = field(default_factory=list)
+
+    @property
+    def complete(self) -> bool:
+        return self.stopped_at is None
+
+
+async def _snapshot(browser: Any, store: MemoryStore) -> Tuple[Any, Dict[int, Dict[str, Any]]]:
+    state = await browser.get_browser_state_summary(include_screenshot=False)
+    snap = store.observe(state.url, state.title, state.dom_state.selector_map or {})
+    return state, snap
+
+
+async def _current_url(browser: Any) -> str:
+    try:
+        return str(await browser.get_current_page_url() or "")
+    except Exception:
+        return ""
+
+
+def _locator_note(target: Optional[Dict[str, Any]], used: Optional[Tuple[str, str]]) -> str:
+    if not target or not used:
+        return ""
+    for loc in target.get("locators") or []:
+        if (loc.get("type"), loc.get("value")) == used:
+            return f"  [{loc['type']} {loc.get('confidence', 0)}%]"
+    return ""
+
+
+def _locator_text(loc: Optional[Tuple[str, str]]) -> str:
+    return f"{loc[0]} {loc[1]}" if loc else "the element's fingerprint in page memory"
+
+
+async def replay_flow(
+    flow: Dict[str, Any],
+    *,
+    browser: Any,
+    store: MemoryStore,
+    emit: Emit,
+    cancel_event: asyncio.Event,
+    on_step: Optional[Callable[[], Awaitable[None]]] = None,
+    overlay: Any = None,
+) -> ReplayOutcome:
+    from browser_use import Tools
+
+    tools = Tools()
+    action_model = tools.registry.create_action_model()
+    steps = [s for s in flow.get("steps") or [] if s.get("action") not in READ_ONLY_ACTIONS]
+    out = ReplayOutcome(total=len(steps))
+
+    def stop(n: int, reason: str) -> ReplayOutcome:
+        out.stopped_at = n
+        out.reason = reason
+        return out
+
+    for n, step in enumerate(steps, 1):
+        if cancel_event.is_set():
+            stop(n, "stopped by user")
+            break
+        action = str(step.get("action") or "")
+        params = dict(step.get("params") or {})
+        line = str(step.get("line") or action)
+        if action not in REPLAYABLE_ACTIONS:
+            stop(n, f"`{line}` needs the agent")
+            break
+        if step.get("secret"):
+            stop(n, f"`{line}` types into a password field that was not saved")
+            break
+        env = params.pop("text_env", None)
+        totp_env = params.pop("totp_env", None)
+        if env or totp_env:
+            if not os.environ.get(env or totp_env):
+                stop(n, f"`{line}` needs {env or totp_env}, which is not set")
+                break
+            params["text"] = totp_code(os.environ[totp_env]) if totp_env else os.environ[env]
+
+        started = time.time()
+        target: Optional[Dict[str, Any]] = None
+        used: Optional[Tuple[str, str]] = None
+        stale: List[Tuple[str, str]] = []
+        snap: Dict[int, Dict[str, Any]] = {}
+        state = None
+        if action in ELEMENT_ACTIONS and (step.get("locators") or step.get("element")):
+            for _ in range(FIND_ATTEMPTS):
+                state, snap = await _snapshot(browser, store)
+                idx, used, stale = resolve_step(step, snap, store)
+                if idx is not None:
+                    target = snap[idx]
+                    params["index"] = idx
+                    break
+                if cancel_event.is_set():
+                    break
+                await asyncio.sleep(FIND_DELAY_SECONDS)
+            if target is None:
+                stop(n, f'could not find "{step.get("label") or line}" on the page any more')
+                break
+
+        url = getattr(state, "url", None) or await _current_url(browser)
+        out.emitted = n
+        emit(
+            {
+                "type": "step",
+                "source": "memory",
+                "step_number": n,
+                "url": url,
+                "title": getattr(state, "title", "") or "",
+                "next_goal": f"Replaying step {n} of {out.total} from memory",
+                "action_lines": [line + _locator_note(target, used)],
+                "status_line": f"Step {n} (memory): {line}",
+            }
+        )
+        if overlay is not None:
+            doing = brand.plain_action(action, dict(step.get("params") or {}), str(step.get("label") or ""))
+            await overlay.acting(f"Step {n} of {out.total} from memory · {doing}", mode="replaying")
+        try:
+            result = await tools.act(action=action_model(**{action: params}), browser_session=browser)
+            err = str(getattr(result, "error", None) or "").strip()
+            detail = err or str(getattr(result, "extracted_content", None) or "").strip()
+        except Exception as e:
+            logger.debug("replay step %s failed", n, exc_info=True)
+            err = detail = f"{type(e).__name__}: {e}"
+        if overlay is not None:
+            await overlay.thinking()
+        ok = not err
+        emit({"type": "result", "step": n, "items": [{"ok": ok, "done": False, "text": (detail.splitlines() or [""])[0][:240]}]})
+        if target:
+            store.record_outcome(target["page"], target["key"], ok, used, stale if ok else ())
+            if ok and step.get("element") and (step.get("page"), step["element"]) != (target["page"], target["key"]):
+                store.record_stale(step.get("page") or target["page"], step["element"], stale)
+        if not ok:
+            stop(n, f"`{line}` failed: {err[:200]}")
+            break
+        recorded = {**params, "text": None, "totp_env": totp_env} if totp_env else params
+        out.done.append(flow_step(action, recorded, target, line) if target else step)
+        out.times.append((started, time.time()))
+        if target:
+            gone = [
+                (l["type"], l["value"]) for l in step.get("locators") or []
+                if not any((l["type"], l["value"]) in e["current"] for e in snap.values())
+            ]
+            if gone:
+                out.heals.append({"step": n, "line": line, "label": step.get("label") or "", "stale": [_locator_text(k) for k in gone], "used": _locator_text(used)})
+        if on_step is not None:
+            await on_step()
+
+    out.url = await _current_url(browser)
+    return out
+
+
+def handoff_task(goal: str, flow: Dict[str, Any], outcome: ReplayOutcome, *, kind: str = "exact") -> str:
+    """
+    The agent's task after a replay: the goal, what already happened, and where to pick up.
+
+    `kind` is "exact" (an earlier successful run of this task), "resume" (the steps that worked in a run of this
+    task that later went wrong), or "interrupted" (the steps a run of this task got through before it was cut off: closed, crashed, or the machine restarted).
+    """
+    when = str(flow.get("last_success") or flow.get("at") or "")[:10]
+    if kind == "resume":
+        source = "an earlier run of this task that went wrong later"
+    elif kind == "interrupted":
+        source = "an earlier run of this task that was cut off before it finished"
+    else:
+        source = "an earlier successful run of this same task"
+    lines = [
+        goal,
+        "",
+        "## Already done (replayed from ForgeQE memory, do not repeat)",
+        f"These steps come from {source}{f' on {when}' if when else ''} and were just replayed in this browser:",
+    ]
+    lines += [f"{i}. {s.get('line') or s.get('action')}" for i, s in enumerate(outcome.done, 1)]
+    if not outcome.complete:
+        lines.append(f"Replay stopped early: {outcome.reason}. The browser is on {outcome.url or 'the current page'}; continue the task from here.")
+    elif kind == "exact":
+        lines.append(
+            f"All recorded steps ran and the browser is on {outcome.url or 'the final page'}. Check this page: "
+            "if the task is complete, call done right away and report the result. Otherwise continue from here."
+        )
+    elif kind == "interrupted":
+        lines.append(
+            f"That is as far as the earlier run got before it was cut off. The browser is on {outcome.url or 'the current page'}. "
+            "Continue the task from here without repeating those steps."
+        )
+    else:
+        lines.append(f"That is as far as the earlier run got on track. The browser is on {outcome.url or 'the current page'}.")
+    if kind == "resume" and flow.get("reason"):
+        lines.append(
+            f"The earlier run then went wrong: {flow['reason']} "
+            "Look at the page before acting, work out why that happened, and take a different approach to the rest of the task."
+        )
+    lines.append(
+        "Values typed or picked during replay came from the earlier run. If the task depends on today's date "
+        '(for example "tomorrow"), check them and fix any that are stale.'
+    )
+    return "\n".join(lines)

package/py/forgeqe_engine/report.py

@@ -0,0 +1,173 @@
+"""
+What a run leaves for the detailed report: a step timeline with screenshots (steps.json, steps/NN.jpg) and, for
+test cases, the agent's own verdict on each numbered test step.
+"""
+from __future__ import annotations
+
+import base64
+import io
+import json
+import logging
+from pathlib import Path
+from typing import Any, Dict, List, Literal, Optional, Sequence, Tuple
+
+from pydantic import BaseModel, Field
+
+from .diagnostics import assign_problems
+
+logger = logging.getLogger(__name__)
+
+SHOT_WIDTH = 1280
+SHOT_QUALITY = 70
+
+
+class CaseStepResult(BaseModel):
+    step: int = Field(description="The test step's number, as numbered in the task")
+    status: Literal["passed", "failed", "not_run"] = Field(
+        description="passed: done and its expected result holds; failed: could not be done or its expected result does not hold; not_run: not reached"
+    )
+    actual: str = Field(description="What the page actually showed for this step, or why it could not be done")
+    cause: Optional[Literal["app", "test_case", "environment", "automation"]] = Field(
+        default=None,
+        description=(
+            "Only for a failed step, why it failed. app: the application does the wrong thing for what the step asks "
+            "(a wrong result, an error message, a missing confirmation): a product defect. test_case: the test case does "
+            "not fit the application: a page, field, or control it names does not exist or is named or placed "
+            "differently, or its expected result describes something else; the test case needs updating, not the app. "
+            "environment: the application or something it needs was unavailable or refused (an error page, a 5xx, a "
+            "network failure, sign-in or test data not available, a bot check). automation: the control is there but you "
+            "could not operate it (it did not respond to clicks or typing), or you ran out of steps."
+        ),
+    )
+
+
+class CaseReport(BaseModel):
+    summary: str = Field(description="The overall result in a few sentences; name the first failing step and what the page showed instead")
+    steps: List[CaseStepResult] = Field(description="One entry per numbered test step, in order")
+
+
+CASE_REPORT_INSTRUCTION = (
+    "## Test step report\n"
+    "This task is a test case with {n} numbered steps. When you finish (success or failure), report every step from 1 to {n}: "
+    "passed when it was done and its expected result holds, failed when it could not be done or its expected result does not hold, "
+    "not_run when you never reached it. For each step say what the page actually showed, and for a failed step its cause: app (a product "
+    "defect), test_case (the test case does not fit the application), environment (the app or something it needs was unavailable), or "
+    "automation (a control that is there could not be operated). Finish with success only if every step passed."
+)
+
+
+def case_report_from_history(history: Any) -> Optional[Dict[str, Any]]:
+    try:
+        report = history.structured_output
+    except Exception as ex:
+        logger.warning("the agent's step report could not be read: %s", ex)
+        return None
+    if report is None:
+        return None
+    data = report.model_dump(mode="json", exclude_none=True)
+    data["steps"] = sorted(data.get("steps") or [], key=lambda s: s.get("step") or 0)
+    return data
+
+
+def _save_shot(b64: str, file: Path) -> bool:
+    try:
+        raw = base64.b64decode(b64.split(",", 1)[-1] + "=" * ((-len(b64)) % 4))
+        from PIL import Image
+
+        img = Image.open(io.BytesIO(raw)).convert("RGB")
+        if img.width > SHOT_WIDTH:
+            img = img.resize((SHOT_WIDTH, int(img.height * SHOT_WIDTH / img.width)))
+        file.parent.mkdir(parents=True, exist_ok=True)
+        img.save(file, "JPEG", quality=SHOT_QUALITY, optimize=True)
+        return True
+    except Exception as ex:
+        logger.debug("step screenshot not saved: %s", ex)
+        return False
+
+
+def build_timeline(
+    out_dir: Path,
+    *,
+    history: Any,
+    collected_steps: List[Dict[str, Any]],
+    replayed: List[Dict[str, Any]],
+    screenshots: str,
+    ok: bool,
+    replay_times: Sequence[Tuple[float, float]] = (),
+    llm_calls: Sequence[Tuple[float, float]] = (),
+    events: Sequence[Dict[str, Any]] = (),
+) -> Optional[str]:
+    """
+    Write steps.json: steps replayed from memory, then each agent step with its actions, where each action landed on
+    the screenshot, timing (and how much of it the model took), the screenshot, and the browser problems during it.
+    """
+    entries: List[Dict[str, Any]] = []
+    for i, s in enumerate(replayed):
+        url = (s.get("params") or {}).get("url") or s.get("page") or ""
+        entry = {"n": len(entries) + 1, "source": "memory", "url": url, "title": "", "goal": "", "actions": [{"text": s.get("line") or s.get("action") or "", "ok": True}]}
+        if i < len(replay_times):
+            t0, t1 = replay_times[i]
+            entry.update(t0=round(t0, 3), ms=int(max(0.0, t1 - t0) * 1000), model_ms=0)
+        entries.append(entry)
+    payloads = {int(p.get("step_number") or 0): p for p in collected_steps}
+    offset = len(replayed)
+    save_shots = screenshots == "all" or (screenshots == "failures" and not ok)
+    for item in list(getattr(history, "history", None) or []):
+        meta = getattr(item, "metadata", None)
+        number = int(getattr(meta, "step_number", 0) or 0)
+        payload = payloads.get(number + offset) or payloads.get(number) or {}
+        state = getattr(item, "state", None)
+        output = getattr(item, "model_output", None)
+        results = list(getattr(item, "result", None) or [])
+        lines = list(payload.get("action_lines") or [])
+        page_url = str(getattr(state, "url", "") or "")
+        if number == 0 and not lines:
+            # The library records its initial actions (a pinned start page) as step 0, not a model decision
+            if not page_url.startswith("http") and not any(getattr(r, "error", None) for r in results):
+                continue
+            lines = [f"open {page_url}"] * len(results)
+            output = None
+        actions = []
+        for i, r in enumerate(results):
+            err = str(getattr(r, "error", None) or "").strip()
+            if getattr(r, "is_done", False):
+                actions.append({"text": "done: " + ("success" if getattr(r, "success", False) else "failure"), "ok": not err, "done": True})
+                continue
+            action = {"text": lines[i] if i < len(lines) else "action", "ok": not err, **({"error": err[:400]} if err else {})}
+            boxes = payload.get("boxes") or []
+            if i < len(boxes) and boxes[i]:
+                action["box"] = boxes[i]
+            actions.append(action)
+        n = len(entries) + 1
+        entry: Dict[str, Any] = {
+            "n": n,
+            "source": "agent",
+            "url": page_url,
+            "title": str(getattr(state, "title", "") or ""),
+            "goal": str(getattr(output, "next_goal", "") or ""),
+            "evaluation": str(getattr(output, "evaluation_previous_goal", "") or ""),
+            "memory": str(getattr(output, "memory", "") or ""),
+            "actions": actions,
+        }
+        try:
+            t0, t1 = float(meta.step_start_time), float(meta.step_end_time)
+            entry["t0"] = round(t0, 3)
+            entry["ms"] = int(max(0.0, t1 - t0) * 1000)
+            entry["model_ms"] = int(sum(max(0.0, min(b, t1) - max(a, t0)) for a, b in llm_calls) * 1000)
+        except Exception:
+            pass
+        if save_shots and state is not None:
+            try:
+                b64 = state.get_screenshot()
+            except Exception:
+                b64 = None
+            rel = f"steps/{n:02d}.jpg"
+            if b64 and _save_shot(b64, out_dir / rel):
+                entry["screenshot"] = rel
+        entries.append(entry)
+    if not entries:
+        return None
+    assign_problems(entries, list(events))
+    file = out_dir / "steps.json"
+    file.write_text(json.dumps(entries, indent=2, ensure_ascii=False), encoding="utf-8")
+    return str(file.resolve())