@capagents/forgeqe 0.0.0-stage → 1.0.1

package/py/forgeqe_engine/__init__.py

@@ -0,0 +1,3 @@
+"""ForgeQE browser engine: the library's Agent plus recovery prompts, loop breaker, and workflow export."""
+
+BROWSER_USE_VERSION = "0.13.10"

package/py/forgeqe_engine/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+raise SystemExit(main())

package/py/forgeqe_engine/brand.py

@@ -0,0 +1,453 @@
+"""
+ForgeQE's look in the browser it drives, when the browser is visible:
+
+- a ForgeQE start page on blank tabs, in place of the library's (which loads a remote logo)
+- on every page, an overlay: a glow round the edge, a bar along the bottom with the step being run, and a lock on the
+  person's mouse and keyboard while the agent thinks, lifted while it acts so its own clicks and typing get through
+
+The overlay is left out of the page the model reads (the library's exclude attribute), hidden from screen readers and the
+accessibility checks (aria-hidden, shadow DOM, outside <body>), and taken out of a screenshot only while its pixels are
+captured. The step bar stays on screen while the agent clicks, types and scrolls. Nothing in it loads a URL, so a page's
+Content-Security-Policy has nothing to block or log.
+"""
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+import os
+import re
+from typing import Any, Dict, Optional, Tuple
+from urllib.parse import urlparse
+
+from .brand_assets import ICON_PNG, MARK_RGBA, MARK_SIZE
+
+logger = logging.getLogger(__name__)
+
+OVERLAY_TAG = "forgeqe-overlay"
+START_ID = "forgeqe-start"
+CDP_TIMEOUT = 3.0
+
+_OVERLAYS: Dict[int, "Overlay"] = {}
+
+# Runs in every top-level page (and on the pages open when the overlay starts). Starts unlocked: a page that loads in
+# the middle of the agent's actions must not block the rest of them. Overlay.update() locks it once the step is over.
+OVERLAY_JS = r"""(() => {
+  if (window.top !== window) return;
+  if (window.__forgeqeOverlay) { window.__forgeqeOverlay.mount(); return; }
+  const CFG = __CONFIG__;
+  const MODES = {
+    thinking: ['Thinking', 'rgba(62,196,255,.16)', '#8fdcff'],
+    acting: ['Acting', 'rgba(52,211,153,.18)', '#6ee7b7'],
+    replaying: ['From memory', 'rgba(167,139,250,.22)', '#c4b5fd'],
+    waiting: ['Your turn', 'rgba(251,191,36,.18)', '#fcd34d'],
+  };
+  // hide counts screenshots in progress. The bar stays up through clicks, typing and scrolling; only a shot takes it down.
+  const state = { text: CFG.text, mode: CFG.mode, lock: false, hide: 0 };
+  let host = null, textEl, chipEl, lockEl, blocker, restores = 0, restoreAt = 0;
+
+  const el = (tag, css, text) => { const e = document.createElement(tag); e.style.cssText = css; if (text) e.textContent = text; return e; };
+  const locked = () => state.lock && !state.hide;
+
+  function paintMark(canvas) {
+    try {
+      const n = CFG.markSize, bin = atob(CFG.mark), px = new Uint8ClampedArray(bin.length);
+      for (let i = 0; i < bin.length; i++) px[i] = bin.charCodeAt(i);
+      canvas.width = canvas.height = n;
+      canvas.getContext('2d').putImageData(new ImageData(px, n, n), 0, 0);
+    } catch (_) {}
+  }
+
+  function build() {
+    host = document.createElement(CFG.tag);
+    host.setAttribute('data-browser-use-exclude', 'true');
+    host.setAttribute('aria-hidden', 'true');
+    // Top layer, so a click that scrolls the page or sets a transform on it cannot carry the bar off screen.
+    try { host.setAttribute('popover', 'manual'); } catch (_) {}
+    host.style.cssText = 'all:initial;';
+    const root = host.attachShadow({ mode: 'open' });
+    const css = document.createElement('style');
+    css.textContent = '*{pointer-events:none!important;}';
+    root.append(css);
+    const glow = el('div', 'position:fixed;inset:0;pointer-events:none;border-radius:6px;' +
+      'box-shadow:inset 0 0 0 2px rgba(50,210,234,.85),inset 0 0 34px 5px rgba(23,87,194,.4);');
+    try { glow.animate([{ opacity: .45 }, { opacity: 1 }], { duration: 1600, direction: 'alternate', iterations: Infinity, easing: 'ease-in-out' }); } catch (_) {}
+    blocker = el('div', 'position:fixed;inset:0;background:transparent;');
+    const bar = el('div', 'position:fixed;left:0;right:0;bottom:0;box-sizing:border-box;display:flex;align-items:center;gap:12px;' +
+      'padding:9px 16px;background:rgba(15,27,61,.96);color:#f4f8ff;border-top:1px solid rgba(50,210,234,.5);' +
+      'box-shadow:0 -8px 24px rgba(0,0,0,.28);font:500 13px/1.4 -apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,Helvetica,Arial,sans-serif;' +
+      'user-select:none;-webkit-user-select:none;');
+    const mark = el('canvas', 'width:26px;height:26px;border-radius:6px;flex:none;display:block;');
+    paintMark(mark);
+    const word = el('span', 'flex:none;font-weight:800;font-size:15px;letter-spacing:-.02em;');
+    word.append(el('span', 'color:#f4f8ff', 'Forge'), el('span', 'color:#3ec4ff', 'QE'));
+    const sep = el('span', 'flex:none;width:1px;height:18px;background:rgba(255,255,255,.22);');
+    textEl = el('span', 'flex:1 1 auto;min-width:0;overflow:hidden;display:-webkit-box;-webkit-line-clamp:2;-webkit-box-orient:vertical;' +
+      'word-break:break-word;color:rgba(244,248,255,.92);');
+    chipEl = el('span', 'flex:none;padding:3px 10px;border-radius:999px;font-size:12px;font-weight:600;white-space:nowrap;');
+    lockEl = el('span', 'flex:none;font-size:12px;color:rgba(244,248,255,.55);white-space:nowrap;', 'Mouse and keyboard locked');
+    bar.append(mark, word, sep, textEl, chipEl, lockEl);
+    root.append(glow, blocker, bar);
+  }
+
+  function apply() {
+    const hidden = state.hide > 0;
+    host.style.setProperty('position', 'fixed', 'important');
+    host.style.setProperty('inset', '0', 'important');
+    host.style.setProperty('margin', '0', 'important');
+    host.style.setProperty('transform', 'none', 'important');
+    host.style.setProperty('contain', 'none', 'important');
+    host.style.setProperty('z-index', '2147483647', 'important');
+    host.style.setProperty('pointer-events', 'none', 'important');
+    host.style.setProperty('visibility', hidden ? 'hidden' : 'visible', 'important');
+    host.style.setProperty('display', hidden ? 'none' : 'block', 'important');
+    try {
+      if (hidden) { if (host.hidePopover && host.matches(':popover-open')) host.hidePopover(); }
+      else if (host.showPopover && !host.matches(':popover-open')) host.showPopover();
+    } catch (_) {}
+    textEl.textContent = state.text || '';
+    const m = MODES[state.mode] || MODES.thinking;
+    chipEl.textContent = m[0];
+    chipEl.style.background = m[1];
+    chipEl.style.color = m[2];
+    blocker.style.setProperty('pointer-events', locked() ? 'auto' : 'none', 'important');
+    blocker.style.cursor = locked() ? 'not-allowed' : 'default';
+    lockEl.style.display = locked() ? '' : 'none';
+    const start = document.getElementById(CFG.startId);
+    if (start) start.style.visibility = hidden ? 'hidden' : 'visible';
+  }
+
+  function mount() {
+    const parent = document.documentElement;
+    if (!parent) return;
+    if (!host) build();
+    if (host.parentNode !== parent) parent.appendChild(host);
+    watch();
+    apply();
+  }
+
+  // A click can rewrite the document. Put the bar back, but stop if the page removes it in a loop.
+  function restore() {
+    const now = Date.now();
+    if (now - restoreAt > 1000) { restores = 0; restoreAt = now; }
+    if (restores > 8) return;
+    restores++;
+    mount();
+  }
+
+  const keep = new MutationObserver(() => { if (host && !host.isConnected && !state.hide) restore(); });
+  function watch() {
+    keep.disconnect();
+    const parent = document.documentElement;
+    if (parent) keep.observe(parent, { childList: true });
+    keep.observe(document, { childList: true });
+  }
+
+  // Registered before the page's own scripts, so the page never sees a person's input while the agent works.
+  const block = (e) => { if (e.isTrusted && locked()) { e.preventDefault(); e.stopImmediatePropagation(); } };
+  for (const t of ['click', 'dblclick', 'auxclick', 'contextmenu', 'mousedown', 'mouseup', 'pointerdown', 'pointerup',
+                   'touchstart', 'touchend', 'wheel', 'dragstart', 'drop', 'keydown', 'keyup', 'keypress', 'beforeinput', 'paste']) {
+    window.addEventListener(t, block, { capture: true, passive: false });
+  }
+
+  window.__forgeqeOverlay = {
+    mount,
+    set(s) { Object.assign(state, s); if (state.hide < 0) state.hide = 0; mount(); },
+    hide() { state.hide++; mount(); },
+    show() { state.hide = Math.max(0, state.hide - 1); mount(); },
+  };
+  if (document.documentElement) mount();
+  document.addEventListener('DOMContentLoaded', mount, { once: true });
+})()"""
+
+START_JS = r"""(() => {
+  if (window.top !== window || location.href !== 'about:blank' || document.getElementById(__ID__)) return;
+  const el = (tag, css, text) => { const e = document.createElement(tag); e.style.cssText = css; if (text) e.textContent = text; return e; };
+  document.title = 'ForgeQE';
+  const icon = document.createElement('link');
+  icon.rel = 'icon';
+  icon.href = __ICON__;
+  (document.head || document.documentElement).appendChild(icon);
+  const page = el('div', 'position:fixed;inset:0;z-index:2147483646;display:flex;flex-direction:column;align-items:center;justify-content:center;gap:20px;' +
+    'background:radial-gradient(circle at 50% 38%,#1b3a7a 0%,#0f1b3d 58%,#0a1330 100%);' +
+    'font-family:-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,Helvetica,Arial,sans-serif;user-select:none;-webkit-user-select:none;');
+  page.id = __ID__;
+  page.setAttribute('data-browser-use-exclude', 'true');
+  page.setAttribute('aria-hidden', 'true');
+  const img = el('img', 'width:128px;height:128px;border-radius:30px;box-shadow:0 18px 50px rgba(0,0,0,.4),0 0 0 1px rgba(255,255,255,.14);');
+  img.src = __ICON__;
+  img.alt = '';
+  const word = el('div', 'font-size:46px;font-weight:800;letter-spacing:-.03em;line-height:1;');
+  word.append(el('span', 'color:#f4f8ff', 'Forge'), el('span', 'color:#3ec4ff', 'QE'));
+  const pill = el('div', 'display:flex;align-items:center;gap:10px;padding:8px 16px;border-radius:999px;background:rgba(255,255,255,.08);' +
+    'border:1px solid rgba(62,196,255,.3);color:rgba(244,248,255,.85);font-size:13px;font-weight:600;');
+  const dot = el('span', 'width:8px;height:8px;border-radius:50%;background:#3ec4ff;box-shadow:0 0 10px #3ec4ff;');
+  try { dot.animate([{ opacity: .3 }, { opacity: 1 }], { duration: 900, direction: 'alternate', iterations: Infinity }); } catch (_) {}
+  pill.append(dot, document.createTextNode(__TEXT__));
+  page.append(img, word, pill);
+  (document.body || document.documentElement).appendChild(page);
+})()"""
+
+_SET_JS = "(() => { const o = window.__forgeqeOverlay; if (o) o.set(%s); })()"
+_HIDDEN_JS = (
+    "(() => { const o = window.__forgeqeOverlay; if (o) o.%s(); "
+    "const s = document.getElementById('" + START_ID + "'); if (s && !o) s.style.visibility = '%s'; })()"
+)
+
+
+class Overlay:
+    """The overlay for one browser. Every call is best effort: a failure is logged, never raised into the run."""
+
+    def __init__(self, browser: Any, *, lock: bool = True, text: str = "ForgeQE is driving this browser") -> None:
+        self.browser = browser
+        self.lock = lock
+        self.state: Dict[str, Any] = {"text": text, "mode": "thinking", "lock": False}
+        # target id -> (script id, source) of the script each page runs on load
+        self._installed: Dict[str, Tuple[str, str]] = {}
+        _OVERLAYS[id(browser)] = self
+
+    def _script(self) -> str:
+        """The overlay as a page loaded now should first show it: current text and mode, unlocked."""
+        cfg = {"tag": OVERLAY_TAG, "startId": START_ID, "text": self.state["text"], "mode": self.state["mode"], "mark": MARK_RGBA, "markSize": MARK_SIZE}
+        return OVERLAY_JS.replace("__CONFIG__", json.dumps(cfg))
+
+    async def thinking(self, text: Optional[str] = None) -> None:
+        await self.update(text=text, mode="thinking", lock=self.lock)
+
+    async def acting(self, text: str, *, mode: str = "acting") -> None:
+        await self.update(text=text, mode=mode, lock=False)
+
+    async def waiting(self, text: str) -> None:
+        await self.update(text=text, mode="waiting", lock=False)
+
+    async def update(self, **changes: Any) -> None:
+        self.state.update({k: v for k, v in changes.items() if v is not None})
+        await self._each_page(_SET_JS % json.dumps(self.state), install=True)
+
+    async def hide(self) -> None:
+        await self._each_page(_HIDDEN_JS % ("hide", "hidden"), install=False)
+
+    async def show(self) -> None:
+        await self._each_page(_HIDDEN_JS % ("show", "visible"), install=False)
+
+    async def start_page(self, target_id: str) -> None:
+        start = (
+            START_JS.replace("__ID__", json.dumps(START_ID))
+            .replace("__ICON__", json.dumps(f"data:image/png;base64,{ICON_PNG}"))
+            .replace("__TEXT__", json.dumps("Starting the browser…"))
+        )
+        try:
+            cdp = await self.browser.get_or_create_cdp_session(target_id=target_id, focus=False)
+            await self._evaluate(cdp, start)
+        except Exception:
+            logger.debug("start page not shown", exc_info=True)
+
+    def close(self) -> None:
+        _OVERLAYS.pop(id(self.browser), None)
+
+    async def _each_page(self, expression: str, *, install: bool) -> None:
+        try:
+            targets = await asyncio.wait_for(self.browser._cdp_get_all_pages(), timeout=CDP_TIMEOUT)
+        except Exception:
+            logger.debug("overlay: pages not listed", exc_info=True)
+            return
+        script = self._script() if install else ""
+        for t in targets or []:
+            target_id = t.get("targetId")
+            try:
+                cdp = await asyncio.wait_for(self.browser.get_or_create_cdp_session(target_id=target_id, focus=False), timeout=CDP_TIMEOUT)
+                if install:
+                    await self._install(cdp, target_id, script)
+                await self._evaluate(cdp, expression)
+            except Exception:
+                logger.debug("overlay: page %s not updated", target_id, exc_info=True)
+
+    async def _install(self, cdp: Any, target_id: str, script: str) -> None:
+        old = self._installed.get(target_id)
+        if old and old[1] == script:
+            return
+        send = cdp.cdp_client.send.Page
+        if old:
+            await asyncio.wait_for(send.removeScriptToEvaluateOnNewDocument(params={"identifier": old[0]}, session_id=cdp.session_id), timeout=CDP_TIMEOUT)
+        added = await asyncio.wait_for(send.addScriptToEvaluateOnNewDocument(params={"source": script}, session_id=cdp.session_id), timeout=CDP_TIMEOUT)
+        self._installed[target_id] = (str(added.get("identifier") or ""), script)
+        if not old:
+            await self._evaluate(cdp, script)
+
+    @staticmethod
+    async def _evaluate(cdp: Any, expression: str) -> None:
+        await asyncio.wait_for(
+            cdp.cdp_client.send.Runtime.evaluate(params={"expression": expression, "awaitPromise": False}, session_id=cdp.session_id),
+            timeout=CDP_TIMEOUT,
+        )
+
+
+def install() -> None:
+    """
+    Patch the library once per process: blank tabs get the ForgeQE start page (or nothing, with no overlay) instead of
+    the library's remote logo, and a screenshot capture hides the overlay only while the pixels are taken.
+    """
+    try:
+        from browser_use.browser.watchdogs.aboutblank_watchdog import AboutBlankWatchdog
+    except ImportError:
+        return
+
+    async def start_page(self: Any, target_id: Any, *args: Any, **kwargs: Any) -> None:
+        overlay = _OVERLAYS.get(id(self.browser_session))
+        if overlay is not None:
+            await overlay.start_page(target_id)
+
+    AboutBlankWatchdog._show_dvd_screensaver_loading_animation_cdp = start_page  # type: ignore[method-assign]
+
+    try:
+        from cdp_use.cdp.page.library import PageClient
+    except ImportError:
+        return
+
+    original_capture = PageClient.captureScreenshot
+    if getattr(original_capture, "__forgeqe__", False):
+        return
+
+    async def captureScreenshot(page: Any, params: Any = None, session_id: Optional[str] = None) -> Any:
+        # Hide for the capture itself, not for the slow work around it, so the step bar stays up while the agent acts.
+        overlays = [o for o in list(_OVERLAYS.values()) if o is not None]
+        for overlay in overlays:
+            await overlay.hide()
+        try:
+            return await original_capture(page, params=params, session_id=session_id)
+        finally:
+            for overlay in overlays:
+                await overlay.show()
+
+    captureScreenshot.__forgeqe__ = True  # type: ignore[attr-defined]
+    PageClient.captureScreenshot = captureScreenshot  # type: ignore[method-assign]
+
+
+def _quoted(value: Any, n: int = 50) -> str:
+    s = re.sub(r"\s+", " ", str(value or "")).strip()
+    return f"“{s if len(s) <= n else s[: n - 1] + '…'}”"
+
+
+def _page_url(url: Any) -> str:
+    try:
+        u = urlparse(str(url or ""))
+    except ValueError:
+        return str(url or "")
+    if not u.hostname:
+        return str(url or "")
+    host = u.hostname[4:] if u.hostname.startswith("www.") else u.hostname
+    path = "" if u.path in ("", "/") else u.path
+    s = host + path
+    return s if len(s) <= 60 else s[:59] + "…"
+
+
+# CSS selector pieces find_elements is given, as the words for what it looks through.
+_SELECTOR_WORDS = (
+    (r"\bbutton\b|role=['\"]?button", "buttons"),
+    (r"(^|[\s,>+~])a(\b|\[|\.|#|:)|role=['\"]?link", "links"),
+    (r"\binput\b|\btextarea\b|role=['\"]?textbox", "input fields"),
+    (r"\bselect\b|\boption\b|role=['\"]?(combobox|listbox|option)", "dropdowns"),
+    (r"\btable\b|\btr\b|\btd\b|\bth\b|role=['\"]?(row|cell|grid)", "table rows"),
+    (r"\bli\b|\bul\b|\bol\b|role=['\"]?(list|listitem)", "list items"),
+    (r"\bh[1-6]\b|role=['\"]?heading", "headings"),
+    (r"\bimg\b|role=['\"]?img", "images"),
+    (r"\bform\b", "forms"),
+    (r"\blabel\b", "labels"),
+)
+
+
+def _selector_words(selector: Any) -> str:
+    sel = str(selector or "")
+    words = [w for pattern, w in _SELECTOR_WORDS if re.search(pattern, sel, re.I)]
+    if not words:
+        return "the elements on the page"
+    return "the page's " + (words[0] if len(words) == 1 else ", ".join(words[:-1]) + " and " + words[-1])
+
+
+def plain_action(name: str, params: Dict[str, Any], label: str = "") -> str:
+    """What one library action does, in words a person reads at a glance: `Clicking “Flights”`."""
+    p = params if isinstance(params, dict) else {}
+    target = f" {_quoted(label)}" if label else ""
+    if name in ("navigate", "go_to_url"):
+        return f"Opening {_page_url(p.get('url'))}" + (" in a new tab" if p.get("new_tab") else "")
+    if name == "search":
+        return f"Searching the web for {_quoted(p.get('query'))}"
+    if name in ("input", "input_text"):
+        text = str(p.get("text") or "")
+        if p.get("text_env") or p.get("totp_env") or "<secret>" in text:
+            return f"Filling in{target or ' a field'}"
+        if not text:
+            return f"Clearing{target or ' a field'}"
+        return f"Typing {_quoted(text, 40)}" + (f" into{target}" if target else "")
+    if name == "click":
+        return f"Clicking{target}" if target else "Clicking on the page"
+    if name == "send_keys":
+        return f"Pressing {p.get('keys') or 'a key'}"
+    if name == "scroll":
+        direction = "down" if p.get("down", True) else "up"
+        return f"Scrolling {direction}" + (f" in{target}" if target else "")
+    if name == "find_text":
+        return f"Scrolling to {_quoted(p.get('text'))}"
+    if name == "record_check":
+        return f"Checking {_quoted(p.get('text'))} on the page"
+    if name == "go_back":
+        return "Going back to the previous page"
+    if name == "wait":
+        return f"Waiting {p.get('seconds') or 3}s for the page to settle"
+    if name == "extract":
+        return f"Reading the page: {str(p.get('query') or 'its content').strip()[:90]}"
+    if name == "search_page":
+        return f"Looking for {_quoted(p.get('pattern'))} on the page"
+    if name == "find_elements":
+        return f"Looking through {_selector_words(p.get('selector'))}"
+    if name == "dropdown_options":
+        return f"Checking the options in{target or ' a dropdown'}"
+    if name == "select_dropdown":
+        return f"Choosing {_quoted(p.get('text'))}" + (f" in{target}" if target else "")
+    if name in ("switch", "switch_tab"):
+        return "Switching to another tab"
+    if name in ("close", "close_tab"):
+        return "Closing a tab"
+    if name == "upload_file":
+        return f"Uploading {os.path.basename(str(p.get('path') or 'a file'))}" + (f" to{target}" if target else "")
+    if name == "screenshot":
+        return "Taking a screenshot"
+    if name == "save_as_pdf":
+        return "Saving the page as a PDF"
+    if name in ("write_file", "replace_file"):
+        return "Writing down what it found"
+    if name == "read_file":
+        return "Reading its notes"
+    if name == "evaluate":
+        return "Running a script on the page"
+    if name == "done":
+        return "Checking the result and finishing"
+    return (name.replace("_", " ").capitalize() + target).strip()
+
+
+# "Next immediate goal: open the sort menu." / "The next immediate goal is to open the sort menu."
+_NEXT_GOAL = re.compile(r"next\s+(?:immediate\s+)?(?:goal|step)\s*(?:is\s+)?(?:to\s+)?[:\-–]?\s*(.+)", re.I | re.S)
+
+
+def step_intent(payload: Dict[str, Any]) -> str:
+    """What the step is for, in one sentence: next_goal, or the next goal the model wrote into its memory."""
+    goal = str(payload.get("next_goal") or "").strip()
+    if not goal:
+        m = _NEXT_GOAL.search(str(payload.get("memory") or ""))
+        goal = m.group(1).strip() if m else ""
+    goal = re.split(r"(?<=[.!?])\s+", re.sub(r"\s+", " ", goal))[0].rstrip(".").strip()
+    if len(goal) > 150:
+        goal = goal[:149].rstrip() + "…"
+    return goal[:1].upper() + goal[1:]
+
+
+def step_text(payload: Dict[str, Any]) -> str:
+    """The bar's line for an agent step: what it is for, then what it is doing, in plain words."""
+    doing = [a for a in payload.get("plain_actions") or [] if a][:3]
+    text = f"Step {payload.get('step_number')}"
+    intent = step_intent(payload)
+    if intent:
+        text += f" · {intent}"
+    if doing:
+        text += " · " + ", then ".join([doing[0]] + [d[:1].lower() + d[1:] for d in doing[1:]])
+    return text[:300]