@capagents/forgeqe 1.3.2 → 2.0.0

package/py/forgeqe_desktop/models.py

@@ -0,0 +1,208 @@
+"""
+The desktop agent's model: the same llms.json profile as the browser engine.
+
+UFO pins its own ``openai`` version, which the browser engine's library does not run with, so the desktop engine calls
+the profile through the ``openai`` SDK directly. Keys, Azure endpoints and Microsoft Entra ID come from the same helpers
+as the browser engine (``forgeqe_engine.llm``), so a profile means the same thing in both. Calls are synchronous: UFO
+calls its model clients synchronously, from inside its own event loop.
+"""
+from __future__ import annotations
+
+import json
+import re
+import time
+from typing import Any, Callable, Dict, List, Mapping, Optional, Tuple
+
+from forgeqe_engine.llm import (
+    AZURE_API_VERSION_DEFAULT,
+    _resolve_api_key,
+    entra_token_provider,
+    normalize_azure_endpoint,
+    uses_entra,
+)
+from forgeqe_engine.model_params import call_with_params, params_for
+
+_DATA_URL = re.compile(r"^data:image/(?:png|jpeg|gif|webp);base64,", re.I)
+
+
+class ProfileError(ValueError):
+    """An llms.json profile that cannot drive the desktop agent."""
+
+
+def pick_profile(llms: Mapping[str, Any], name: Optional[str] = None) -> Tuple[str, Dict[str, Any]]:
+    """
+    ``(name, profile)`` from a parsed llms.json: the one asked for, else ``default``, else the first, the way the CLI
+    picks it. Mock profiles are skipped.
+    """
+    models = llms.get("models") if isinstance(llms, Mapping) else None
+    if not isinstance(models, Mapping):
+        raise ProfileError("llms.json has no models")
+    profiles = {k: dict(v) for k, v in models.items() if isinstance(v, Mapping) and k != "mock" and v.get("provider") != "mock"}
+    if name:
+        if name not in profiles:
+            raise ProfileError(f'Unknown LLM profile "{name}". Run: forgeqe models')
+        return name, profiles[name]
+    default = llms.get("default")
+    if isinstance(default, str) and default in profiles:
+        return default, profiles[default]
+    if not profiles:
+        raise ProfileError("llms.json has no profiles")
+    first = next(iter(profiles))
+    return first, profiles[first]
+
+
+def load_profile(llms_path: str, name: Optional[str] = None) -> Tuple[str, Dict[str, Any]]:
+    with open(llms_path, "r", encoding="utf-8") as f:
+        return pick_profile(json.load(f), name)
+
+
+def takes_images(profile: Mapping[str, Any]) -> bool:
+    """``vision`` when the profile sets it; otherwise yes, except for ollama, whose local models are mostly text-only."""
+    if "vision" in profile:
+        return profile.get("vision") is True
+    return str(profile.get("provider") or "azure").lower() != "ollama"
+
+
+def require_vision(name: str, profile: Mapping[str, Any]) -> None:
+    if not takes_images(profile):
+        raise ProfileError(
+            f'llms.json profile "{name}" does not take images, and the desktop agent sends screenshots. '
+            'Pick a vision model, or set "vision": true on the profile if it does take them.'
+        )
+
+
+def check_messages(messages: List[Mapping[str, Any]]) -> List[Dict[str, Any]]:
+    """
+    OpenAI-style chat messages as they go to the model. Images must be inline ``data:`` screenshots: a link would make
+    the model provider fetch a URL that the app under test, or text injected into it, chose.
+    """
+    out: List[Dict[str, Any]] = []
+    for m in messages:
+        if not isinstance(m, Mapping):
+            raise ValueError("each message must be a mapping")
+        content = m.get("content")
+        if isinstance(content, list):
+            for p in content:
+                if isinstance(p, Mapping) and p.get("type") in ("image_url", "input_image"):
+                    raw = p.get("image_url")
+                    url = raw.get("url") if isinstance(raw, Mapping) else raw
+                    if not _DATA_URL.match(str(url or "")):
+                        raise ValueError("images must be inline data: URLs (png, jpeg, gif or webp)")
+        out.append(dict(m))
+    return out
+
+
+def build_client(profile: Mapping[str, Any]) -> Tuple[Any, str]:
+    """An ``openai`` client and the model (or Azure deployment) name for a profile."""
+    import openai
+
+    cfg = dict(profile)
+    provider = str(cfg.get("provider") or "azure").lower()
+    entra = uses_entra(cfg)
+    if entra and provider not in ("azure", "openai_compatible"):
+        raise ProfileError(f"auth: entra works with the azure and openai_compatible providers, not {provider}.")
+    api_key = "" if entra else _resolve_api_key(cfg)
+    model = str(cfg.get("model") or "gpt-4o")
+
+    if provider == "azure":
+        endpoint = normalize_azure_endpoint(cfg.get("endpoint") or cfg.get("apiEndpoint") or "")
+        deployment = str(cfg.get("deployment") or cfg.get("deploymentName") or model)
+        api_version = str(cfg.get("api_version") or cfg.get("apiVersion") or AZURE_API_VERSION_DEFAULT)
+        if not endpoint:
+            raise ProfileError("Azure OpenAI needs an endpoint: check it in llms.json.")
+        kw: Dict[str, Any] = {"azure_endpoint": endpoint, "api_version": api_version}
+        if entra:
+            kw["azure_ad_token_provider"] = entra_token_provider(cfg)
+        elif api_key:
+            kw["api_key"] = api_key
+        else:
+            raise ProfileError(
+                "Azure OpenAI needs an API key: set the profile's api_key_env variable (see llms.json), "
+                "or set auth: entra to sign in with Microsoft Entra ID instead."
+            )
+        return openai.AzureOpenAI(**kw), deployment
+
+    if provider == "openai":
+        if not api_key:
+            raise ProfileError("OpenAI needs an API key: set OPENAI_API_KEY or the profile's api_key_env.")
+        return openai.OpenAI(api_key=api_key), model
+
+    if provider in ("openai_compatible", "ollama"):
+        base = str(cfg.get("endpoint") or cfg.get("apiEndpoint") or ("http://127.0.0.1:11434/v1" if provider == "ollama" else "")).strip()
+        if not base:
+            raise ProfileError("openai_compatible profiles need an endpoint (base URL).")
+        if provider == "ollama" and not re.search(r"/v1/?$", base):
+            base = base.rstrip("/") + "/v1"
+        if entra:
+            import httpx
+
+            token = entra_token_provider(cfg)
+
+            class _Bearer(httpx.Auth):
+                def auth_flow(self, request: httpx.Request):
+                    request.headers["Authorization"] = f"Bearer {token()}"
+                    yield request
+
+            return openai.OpenAI(base_url=base, api_key="entra", http_client=httpx.Client(auth=_Bearer())), model
+        return openai.OpenAI(base_url=base, api_key=api_key or "local"), model
+
+    raise ProfileError(f"Unsupported LLM provider for the desktop engine: {provider}")
+
+
+def _usage(response: Any) -> Dict[str, int]:
+    u = getattr(response, "usage", None)
+    if u is None:
+        return {}
+    out = {"input": int(getattr(u, "prompt_tokens", 0) or 0), "output": int(getattr(u, "completion_tokens", 0) or 0)}
+    details = getattr(u, "prompt_tokens_details", None)
+    cached = int(getattr(details, "cached_tokens", 0) or 0) if details is not None else 0
+    if cached:
+        out["cached"] = cached
+    return out
+
+
+class DesktopModel:
+    """A profile's chat model; each call's time and tokens go to ``on_call``, never the prompt or the reply."""
+
+    def __init__(
+        self,
+        profile: Mapping[str, Any],
+        on_call: Optional[Callable[[Dict[str, Any]], None]] = None,
+        client: Any = None,
+        model: Optional[str] = None,
+        timeout: float = 120.0,
+    ) -> None:
+        if client is None:
+            client, built_model = build_client(profile)
+            model = model or built_model
+        self.client = client
+        self.model = model or str(profile.get("deployment") or profile.get("model") or "gpt-4o")
+        self.temperature = profile.get("temperature")
+        self.on_call = on_call
+        self.timeout = timeout
+        # The parameters this model takes (token limit's name, temperature, ...), shared with every other call to it.
+        self.params = params_for(profile)
+
+    def complete(self, messages: List[Mapping[str, Any]], max_tokens: Optional[int] = None, json_mode: bool = False) -> str:
+        kw: Dict[str, Any] = {"model": self.model, "messages": check_messages(messages), "timeout": self.timeout}
+        if isinstance(self.temperature, (int, float)):
+            kw["temperature"] = self.temperature
+        if max_tokens:
+            kw["max_tokens"] = max_tokens
+        if json_mode:
+            kw["response_format"] = {"type": "json_object"}
+        started = time.time()
+        response: Any = None
+        ok = False
+        try:
+            response = call_with_params(self.params, self.client.chat.completions.create, kw)
+            ok = True
+        finally:
+            if self.on_call is not None:
+                try:
+                    self.on_call({"ms": int((time.time() - started) * 1000), "ok": ok, **_usage(response)})
+                except Exception:
+                    pass
+        choices = getattr(response, "choices", None) or []
+        content = getattr(getattr(choices[0], "message", None), "content", "") if choices else ""
+        return content or ""

package/py/forgeqe_desktop/pywinauto_driver.py

@@ -0,0 +1,217 @@
+"""
+The Windows driver for replay: pywinauto with the UIA backend. Windows only; the module imports anywhere so the key
+translation can be tested on any OS.
+
+The app is started with ``subprocess.Popen([path, *args])``, no shell, and only controls inside that app's own windows
+are searched. Typed text is escaped so pywinauto never reads it as key presses.
+"""
+from __future__ import annotations
+
+import subprocess
+import time
+from typing import Any, Dict, List, Mapping, Optional
+
+from .apps import AppSpec
+from .replay import ControlNotFound
+from .workflow import MODIFIERS, parse_chord
+
+_SEND_KEYS_SPECIAL = set("+^%~(){}[]")
+_SEND_KEYS_MODS = {"ctrl": "^", "shift": "+", "alt": "%"}
+_SEND_KEYS_NAMES = {
+    "enter": "ENTER", "tab": "TAB", "esc": "ESC", "space": "SPACE", "backspace": "BACKSPACE", "delete": "DELETE",
+    "insert": "INSERT", "home": "HOME", "end": "END", "pageup": "PGUP", "pagedown": "PGDN",
+    "up": "UP", "down": "DOWN", "left": "LEFT", "right": "RIGHT", **{f"f{n}": f"F{n}" for n in range(1, 13)},
+}
+
+
+def escape_send_keys(text: str) -> str:
+    """``text`` for pywinauto's ``type_keys`` so each character is typed as itself: ``^{ESC}`` types ``^{ESC}``."""
+    return "".join("{%s}" % c if c in _SEND_KEYS_SPECIAL else c for c in text)
+
+
+def to_send_keys(chord: List[str]) -> str:
+    """A parsed chord (``["ctrl", "s"]``) in pywinauto's key syntax (``"^s"``)."""
+    parse_chord("+".join(chord))
+    *mods, key = chord
+    prefix = "".join(_SEND_KEYS_MODS[m] for m in mods if m in MODIFIERS)
+    name = _SEND_KEYS_NAMES.get(key)
+    return prefix + ("{%s}" % name if name else escape_send_keys(key))
+
+
+def _criteria(target: Mapping[str, Any]) -> Dict[str, Any]:
+    crit: Dict[str, Any] = {}
+    if target.get("automation_id"):
+        crit["auto_id"] = target["automation_id"]
+    if target.get("name"):
+        crit["title"] = target["name"]
+    if target.get("control_type"):
+        crit["control_type"] = target["control_type"]
+    if target.get("class_name"):
+        crit["class_name"] = target["class_name"]
+    if "index" in target:
+        crit["found_index"] = int(target["index"])
+    return crit
+
+
+class PywinautoDriver:
+    # Drives the real desktop, so a run first checks the session can take input (session.py).
+    needs_desktop = True
+
+    def __init__(self, launch_timeout: float = 30.0, poll: float = 0.25) -> None:
+        self.launch_timeout = launch_timeout
+        self.poll = poll
+        self.app: Any = None
+        self.proc: Optional[subprocess.Popen] = None
+
+    def launch(self, app: AppSpec, timeout: Optional[float] = None) -> None:
+        from pywinauto import Application
+
+        wait = self.launch_timeout if timeout is None else timeout
+        self.proc = subprocess.Popen(app.argv(), shell=False)
+        if app.window:
+            # A launcher that starts the real app and exits: connect by the main window's title instead.
+            self.app = Application(backend="uia").connect(title_re=app.window, timeout=wait)
+        else:
+            self.app = Application(backend="uia").connect(process=self.proc.pid, timeout=wait)
+
+    def _require_app(self) -> Any:
+        if self.app is None:
+            raise RuntimeError("the app is not running: the workflow needs a launch step first")
+        return self.app
+
+    def find(self, target: Mapping[str, Any], timeout: float) -> Any:
+        from pywinauto.findwindows import ElementAmbiguousError, ElementNotFoundError
+
+        app = self._require_app()
+        deadline = time.time() + timeout
+        last = "no window"
+        while True:
+            for win in app.windows():
+                spec = app.window(handle=win.handle)
+                for part in target.get("path") or []:
+                    spec = spec.child_window(**{k: v for k, v in (("control_type", part.get("control_type")), ("title", part.get("name"))) if v})
+                try:
+                    return spec.child_window(**_criteria(target)).wrapper_object()
+                except ElementAmbiguousError:
+                    raise ControlNotFound(f"several controls match {dict(target)}; add an index or more properties")
+                except ElementNotFoundError as e:
+                    last = str(e) or "not found"
+            if time.time() >= deadline:
+                raise ControlNotFound(f"no control matches {dict(target)} ({last})")
+            time.sleep(self.poll)
+
+    def wait_ready(self, control: Any, timeout: float) -> None:
+        """Wait until the control is visible and enabled (a form still loading greys its buttons out)."""
+        deadline = time.time() + timeout
+        while True:
+            try:
+                if control.is_visible() and control.is_enabled():
+                    return
+            except Exception:
+                return
+            if time.time() >= deadline:
+                raise TimeoutError(f"the control is on screen but stayed disabled or hidden for {timeout:g} s")
+            time.sleep(self.poll)
+
+    def click(self, control: Any, double: bool = False) -> None:
+        if double:
+            control.double_click_input()
+        else:
+            control.click_input()
+
+    def type_text(self, control: Any, text: str) -> None:
+        control.set_focus()
+        setter = getattr(control, "set_edit_text", None)
+        if setter is not None:
+            try:
+                setter(text)
+                return
+            except Exception:
+                pass
+        control.type_keys(escape_send_keys(text), with_spaces=True, with_tabs=True, with_newlines=True, set_foreground=True)
+
+    def select(self, control: Any, value: str) -> None:
+        control.select(value)
+
+    def press(self, chord: List[str]) -> None:
+        from pywinauto.keyboard import send_keys
+
+        app = self._require_app()
+        app.top_window().set_focus()
+        send_keys(to_send_keys(chord))
+
+    def text_of(self, control: Any) -> str:
+        texts = [str(control.window_text() or "")]
+        try:
+            texts.append(str(control.get_value() or ""))
+        except Exception:
+            pass
+        return "\n".join(t for t in texts if t)
+
+    def describe(self, limit: int = 400, texts: bool = False) -> List[Dict[str, str]]:
+        """
+        The UIA properties of the controls in the app's windows, at most ``limit``: for healing a step, and with
+        ``texts`` the text each control shows, for choosing checks.
+        """
+        app = self._require_app()
+        out: List[Dict[str, str]] = []
+        for win in app.windows():
+            for c in [win, *win.descendants()]:
+                info = c.element_info
+                props = {k: v for k, v in (("automation_id", info.automation_id), ("name", info.name), ("control_type", info.control_type), ("class_name", info.class_name)) if isinstance(v, str) and v}
+                if props and texts:
+                    try:
+                        text = self.text_of(c)
+                    except Exception:
+                        text = ""
+                    if text:
+                        props["text"] = text
+                if props:
+                    out.append(props)
+                if len(out) >= limit:
+                    return out
+        return out
+
+    def image_png(self, control: Any = None) -> Optional[bytes]:
+        """The control, or the app's top window, as PNG for expect_image; None when it cannot be captured."""
+        import io
+
+        try:
+            target = control if control is not None else self._require_app().top_window().wrapper_object()
+            image = target.capture_as_image()
+            if image is None:
+                return None
+            buf = io.BytesIO()
+            image.save(buf, format="PNG")
+            return buf.getvalue()
+        except Exception:
+            return None
+
+    def screenshot_png(self) -> Optional[bytes]:
+        """The app's top window as PNG, or None when it cannot be captured (minimised, locked screen)."""
+        import io
+
+        if self.app is None:
+            return None
+        try:
+            image = self.app.top_window().wrapper_object().capture_as_image()
+            buf = io.BytesIO()
+            image.save(buf, format="PNG")
+            return buf.getvalue()
+        except Exception:
+            return None
+
+    def close(self) -> None:
+        """Close the app this driver started, and only that one: a polite close first, then terminate."""
+        if self.app is not None:
+            try:
+                self.app.kill(soft=True)
+            except Exception:
+                pass
+        if self.proc is not None and self.proc.poll() is None:
+            try:
+                self.proc.wait(timeout=5)
+            except subprocess.TimeoutExpired:
+                self.proc.terminate()
+        self.app = None
+        self.proc = None

package/py/forgeqe_desktop/record.py

@@ -0,0 +1,150 @@
+"""
+A desktop run's trace as workflow steps.
+
+The UFO adapter normalises each action UFO took to ``{"action", "control", "args", "ok"}``, where ``control`` holds the
+UIA properties of the control it acted on. Failed actions are left out; actions that have no step form are returned
+in ``skipped`` so the run report can show them. A typed value equal to a known secret is written as its placeholder.
+"""
+from __future__ import annotations
+
+import re
+from typing import Any, Dict, List, Mapping, Optional, Tuple
+
+from .workflow import FORMAT, LOCATOR_KEYS, WorkflowError, parse_chord, validate
+
+_CONTROL_KEYS = {
+    "automation_id": ("automation_id", "auto_id", "AutomationId", "automationId"),
+    "name": ("name", "title", "Name", "control_text"),
+    "control_type": ("control_type", "ControlType", "controlType"),
+    "class_name": ("class_name", "ClassName", "className", "control_class"),
+}
+_PYWINAUTO_NAMES = {
+    "enter": "enter", "tab": "tab", "esc": "esc", "escape": "esc", "space": "space",
+    "backspace": "backspace", "bs": "backspace", "bksp": "backspace", "delete": "delete", "del": "delete",
+    "insert": "insert", "ins": "insert", "home": "home", "end": "end", "pgup": "pageup", "pgdn": "pagedown",
+    "up": "up", "down": "down", "left": "left", "right": "right",
+    **{f"f{n}": f"f{n}" for n in range(1, 13)},
+}
+_PYWINAUTO_MODS = {"^": "ctrl", "+": "shift", "%": "alt"}
+_PYWINAUTO_CHORD = re.compile(r"^([\^+%]*)(\{[A-Za-z0-9_]+\}|[^\^+%~{}()\[\]])$")
+
+
+def target_of(control: Any) -> Optional[Dict[str, Any]]:
+    """The locator for a control's UIA properties, or None when it has neither an AutomationId nor a name."""
+    if not isinstance(control, Mapping):
+        return None
+    target: Dict[str, Any] = {}
+    for key, names in _CONTROL_KEYS.items():
+        for n in names:
+            value = control.get(n)
+            if isinstance(value, str) and value.strip():
+                target[key] = value.strip()
+                break
+    if not target.get("automation_id") and not target.get("name"):
+        return None
+    return {k: target[k] for k in LOCATOR_KEYS if k in target}
+
+
+def chord_from_pywinauto(keys: str) -> Optional[str]:
+    """``"^s"`` as ``"ctrl+s"``, ``"%{F4}"`` as ``"alt+f4"``; None when it is not one chord the workflow allows."""
+    m = _PYWINAUTO_CHORD.match((keys or "").strip())
+    if not m:
+        return None
+    mods = [_PYWINAUTO_MODS[c] for c in m.group(1)]
+    key = m.group(2)
+    if key.startswith("{"):
+        key = _PYWINAUTO_NAMES.get(key[1:-1].lower(), "")
+        if not key:
+            return None
+    chord = "+".join([*mods, key.lower()])
+    try:
+        parse_chord(chord)
+    except WorkflowError:
+        return None
+    return chord
+
+
+def _mask(text: str, secrets: Mapping[str, str]) -> str:
+    # Longest first, so a secret that contains another is replaced whole.
+    for name, value in sorted(secrets.items(), key=lambda kv: -len(kv[1] or "")):
+        if value:
+            text = text.replace(value, "{{secret:%s}}" % name)
+    return text
+
+
+def steps_from_trace(
+    trace: List[Mapping[str, Any]],
+    app: str,
+    name: str = "",
+    secrets: Optional[Mapping[str, str]] = None,
+) -> Tuple[Dict[str, Any], List[Dict[str, Any]]]:
+    """``(workflow, skipped)`` for a trace; the workflow starts by launching ``app`` and is validated before return."""
+    secrets = dict(secrets or {})
+    steps: List[Dict[str, Any]] = [{"op": "launch"}]
+    skipped: List[Dict[str, Any]] = []
+    variables: Dict[str, str] = {}
+
+    def literal(text: str) -> Optional[str]:
+        """
+        Text for a step, secrets masked. Text that contains ``{{…}}`` (a model told to type ``{{secret:X}}``) is stored
+        as a variable's value, which replay types as it is; None when it also holds a secret and can be neither.
+        """
+        masked = _mask(text, secrets)
+        if "{{" not in text and "}}" not in text:
+            return masked
+        if masked != text:
+            return None
+        var = f"text_{len(variables) + 1}"
+        variables[var] = text
+        return "{{%s}}" % var
+
+    for i, entry in enumerate(trace, 1):
+        if not isinstance(entry, Mapping) or entry.get("ok") is False:
+            continue
+        action = str(entry.get("action") or "").strip().lower()
+        args = entry.get("args") if isinstance(entry.get("args"), Mapping) else {}
+        target = target_of(entry.get("control"))
+
+        def skip(reason: str) -> None:
+            skipped.append({"index": i, "action": action, "reason": reason})
+
+        if action in ("click_input", "click", "double_click"):
+            if target is None:
+                skip("the control has no AutomationId or name")
+                continue
+            double = action == "double_click" or bool(args.get("double"))
+            steps.append({"op": "double_click" if double else "click", "target": target})
+        elif action in ("set_edit_text", "type", "input_text"):
+            text = args.get("text")
+            if target is None or not isinstance(text, str):
+                skip("no control or no text")
+                continue
+            filled = literal(text)
+            if filled is None:
+                skip("text with both a secret and {{…}}")
+                continue
+            steps.append({"op": "type", "target": target, "text": filled})
+        elif action in ("select", "select_item"):
+            value = args.get("value", args.get("text"))
+            if target is None or not isinstance(value, str):
+                skip("no control or no value")
+                continue
+            filled = literal(value)
+            if filled is None:
+                skip("value with both a secret and {{…}}")
+                continue
+            steps.append({"op": "select", "target": target, "value": filled})
+        elif action in ("keyboard_input", "keys", "type_keys"):
+            keys = str(args.get("keys") or args.get("text") or "")
+            chord = chord_from_pywinauto(keys)
+            if chord:
+                steps.append({"op": "keys", "keys": chord})
+            elif target is not None and keys and not any(c in keys for c in "^+%~{}()[]"):
+                steps.append({"op": "type", "target": target, "text": _mask(keys, secrets)})
+            else:
+                skip("keys that are not one allowed chord")
+        else:
+            skip("no workflow step for this action")
+
+    workflow = {"format": FORMAT, "name": name, "app": app, "variables": variables, "steps": steps}
+    return validate(workflow), skipped