@capagents/forgeqe 1.3.2 → 2.0.0

package/py/forgeqe_desktop/service.py

@@ -0,0 +1,356 @@
+"""
+``python -m forgeqe_desktop serve``: one request as a JSON line on stdin, events as JSON lines on stdout, like the
+browser engine. A later ``stop`` line on stdin stops a replay between steps, or ends a recording.
+
+Request: ``mode`` (``replay``, ``explore`` or ``record``), ``out_dir``, ``desktop`` (``apps``, ``secrets``,
+``ufo_dir``), ``llm`` and ``profile_name`` (the llms.json profile, for healing and explore), and per mode: ``workflow``,
+``variables``, ``heal``, ``screenshots``, ``update_baselines`` (replay); ``goal``, ``app``, ``name`` (explore); ``app``,
+``name`` (record: the tester does the steps by hand).
+
+Events: ``log``, ``step`` (one per replayed step), ``recorded`` (one per step recorded by hand), ``call`` (one model
+call's time and tokens, never its text), and ``done`` with the result. ``run.json`` in ``out_dir`` holds the same result.
+"""
+from __future__ import annotations
+
+import json
+import os
+import sys
+import threading
+import time
+from pathlib import Path
+from typing import IO, Any, Callable, Dict, List, Mapping, Optional
+
+from .apps import load_apps, resolve_app
+from .workflow import FORMAT, WorkflowError, load, save, validate
+
+
+class Events:
+    def __init__(self, out: IO[str]) -> None:
+        self.out = out
+        self.lock = threading.Lock()
+
+    def emit(self, event: Dict[str, Any]) -> None:
+        with self.lock:
+            self.out.write(json.dumps(event, ensure_ascii=False, default=str) + "\n")
+            self.out.flush()
+
+    def log(self, text: str, level: str = "info") -> None:
+        self.emit({"type": "log", "level": level, "text": text})
+
+
+def _secret_names(desktop: Mapping[str, Any]) -> List[str]:
+    names = desktop.get("secrets") or []
+    if not isinstance(names, list) or not all(isinstance(n, str) for n in names):
+        raise WorkflowError("desktop.secrets must be a list of environment variable names")
+    return names
+
+
+def _out_dir(req: Mapping[str, Any]) -> Path:
+    raw = req.get("out_dir")
+    if not isinstance(raw, str) or not raw:
+        raise WorkflowError("out_dir is required")
+    out = Path(raw)
+    out.mkdir(parents=True, exist_ok=True)
+    return out
+
+
+def _model(req: Mapping[str, Any], events: Events, model_factory: Optional[Callable[..., Any]]) -> Any:
+    from .models import DesktopModel
+
+    profile = req.get("llm")
+    if not isinstance(profile, dict):
+        raise WorkflowError("this mode needs an llms.json profile")
+    on_call = lambda c: events.emit({"type": "call", **c})  # noqa: E731
+    return (model_factory or DesktopModel)(profile, on_call=on_call)
+
+
+def _step_fields(res: Any) -> Dict[str, Any]:
+    out = {"index": res.index, "op": res.op, "ok": res.ok, "ms": res.ms, "error": res.error, "healed_target": res.healed_target}
+    if res.attempts > 1:
+        out["attempts"] = res.attempts
+    if res.dismissed:
+        out["dismissed"] = list(res.dismissed)
+    if res.images:
+        out["images"] = dict(res.images)
+    if res.note:
+        out["note"] = res.note
+    return out
+
+
+def _check_desktop(driver: Any) -> None:
+    """Fail before the app starts when this Windows session cannot take input (a service, a locked screen)."""
+    if sys.platform != "win32" or not getattr(driver, "needs_desktop", False):
+        return
+    from .session import desktop_problem
+
+    problem = desktop_problem()
+    if problem:
+        raise WorkflowError(f"This Windows session cannot be driven: {problem}")
+
+
+def run_replay(req: Mapping[str, Any], events: Events, driver: Any, stop: threading.Event, model_factory: Optional[Callable[..., Any]] = None) -> Dict[str, Any]:
+    from .heal import make_heal
+    from .models import takes_images
+    from .replay import replay
+
+    out = _out_dir(req)
+    desktop = req.get("desktop") or {}
+    workflow_file = str(req.get("workflow") or "")
+    wf = load(workflow_file)
+    heal = make_heal(_model(req, events, model_factory), driver, log=events.log, images=takes_images(req.get("llm") or {})) if req.get("heal") else None
+    shots = bool(req.get("screenshots", True))
+    _check_desktop(driver)
+    from .visual import image_checker
+
+    compare_image = image_checker(workflow_file, out, update=bool(req.get("update_baselines")))
+    (out / "steps").mkdir(exist_ok=True)
+
+    def on_step(res: Any) -> None:
+        shot = None
+        grab = getattr(driver, "screenshot_png", None)
+        if shots and grab is not None and res.op not in ("close",):
+            png = grab()
+            if png:
+                shot = f"steps/{res.index:03d}.png"
+                (out / shot).write_bytes(png)
+        events.emit({"type": "step", **_step_fields(res), "screenshot": shot})
+
+    result = replay(
+        wf,
+        driver,
+        load_apps({"desktop": desktop}),
+        variables=req.get("variables") or {},
+        allowed_secrets=_secret_names(desktop),
+        heal=heal,
+        on_step=on_step,
+        should_stop=stop.is_set,
+        compare_image=compare_image,
+    )
+    steps = []
+    for s in result.steps:
+        shot = f"steps/{s.index:03d}.png"
+        steps.append({**_step_fields(s), "screenshot": shot if (out / shot).is_file() else None, "target": wf["steps"][s.index - 1].get("target")})
+    return {
+        "mode": "replay",
+        "ok": result.ok,
+        "stopped": result.stopped,
+        "name": wf["name"],
+        "app": wf["app"],
+        "workflow": workflow_file,
+        "total": len(wf["steps"]),
+        "steps": steps,
+        "heals": [{"index": s.index, "old": wf["steps"][s.index - 1].get("target"), "new": s.healed_target} for s in result.healed],
+    }
+
+
+def _ufo_verdict(out: Path) -> Optional[bool]:
+    """UFO's own evaluation of the run (``evaluation.log``), when it wrote one."""
+    path = out / "ufo" / "evaluation.log"
+    if not path.is_file():
+        return None
+    for line in reversed(path.read_text(encoding="utf-8", errors="replace").splitlines()):
+        try:
+            doc = json.loads(line)
+        except ValueError:
+            continue
+        complete = str(doc.get("complete", "")).strip().lower() if isinstance(doc, dict) else ""
+        if complete in ("yes", "no"):
+            return complete == "yes"
+    return None
+
+
+def run_explore(
+    req: Mapping[str, Any],
+    events: Events,
+    driver: Any,
+    runner: Optional[Callable[..., List[Dict[str, Any]]]] = None,
+    model_factory: Optional[Callable[..., Any]] = None,
+) -> Dict[str, Any]:
+    from .record import steps_from_trace
+    from .models import require_vision
+
+    out = _out_dir(req)
+    desktop = req.get("desktop") or {}
+    goal = str(req.get("goal") or "").strip()
+    if not goal:
+        raise WorkflowError("explore needs a goal")
+    app = str(req.get("app") or "")
+    spec = resolve_app(app, load_apps({"desktop": desktop}))
+    profile = req.get("llm") or {}
+    profile_name = str(req.get("profile_name") or "profile")
+    require_vision(profile_name, profile)
+    name = str(req.get("name") or goal)[:120]
+    ufo_dir = Path(str(desktop.get("ufo_dir") or ""))
+    secrets = {n: os.environ[n] for n in _secret_names(desktop) if os.environ.get(n)}
+
+    if runner is None:
+        from .ufo_runner import run_ufo as runner
+
+    _check_desktop(driver)
+    events.log(f"Starting {app} and handing the goal to UFO.")
+    driver.launch(spec)
+    started = time.time()
+    trace: List[Dict[str, Any]] = []
+    ufo_error = ""
+    try:
+        runner(
+            ufo_dir,
+            goal,
+            task=f"forgeqe-{out.name}",
+            profile_name=profile_name,
+            profile=profile,
+            out_dir=out,
+            on_call=lambda c: events.emit({"type": "call", **c}),
+            log=events.log,
+            trace=trace,
+            **({"model": model_factory(profile, on_call=lambda c: events.emit({"type": "call", **c}))} if model_factory else {}),
+        )
+    except Exception as e:
+        # UFO stopping with an error (its evaluation, a parse failure) still leaves the actions it took in the trace.
+        ufo_error = f"UFO stopped with an error: {type(e).__name__}: {e}"[:2000]
+        events.log(ufo_error, "error")
+    workflow, skipped = steps_from_trace(trace, app=app, name=name, secrets=secrets)
+    recorded = len(workflow["steps"]) - 1
+    checks: List[Dict[str, Any]] = []
+    values: Dict[str, str] = {}
+    try:
+        # Checks are read from the app as UFO left it, before it closes: they make the replay test the outcome.
+        if recorded and not ufo_error and req.get("checks", True):
+            from .checks import read_outcome
+
+            checks, values = read_outcome(_model(req, events, model_factory), driver, goal, secrets=secrets, log=events.log)
+            if checks:
+                workflow = validate({**workflow, "steps": [*workflow["steps"], *checks]})
+                events.log(f"Added {len(checks)} check{'' if len(checks) == 1 else 's'} of the outcome.")
+            else:
+                events.log("No control shows text that proves the goal; the workflow has no checks. Add expect steps by hand.", "warn")
+    except Exception as e:
+        events.log(f"Checks were not added: {e}", "warn")
+    finally:
+        try:
+            driver.close()
+        except Exception:
+            pass
+    (out / "trace.json").write_text(json.dumps(trace, ensure_ascii=False, indent=2, default=str), encoding="utf-8")
+    verdict = _ufo_verdict(out)
+    workflow_file = out / "workflow.yaml"
+    if recorded:
+        save(workflow, workflow_file)
+    ok = recorded > 0 and verdict is not False and not ufo_error
+    return {
+        "mode": "explore",
+        "ok": ok,
+        "goal": goal,
+        "app": app,
+        "name": name,
+        "seconds": int(time.time() - started),
+        "recorded": recorded,
+        "checks": len(checks),
+        "values": values,
+        "skipped": skipped,
+        "refused": [t for t in trace if t.get("refused")],
+        "ufo_complete": verdict,
+        "workflow": str(workflow_file) if recorded else None,
+        "error": ufo_error + (f" The workflow holds the {recorded} steps recorded before that." if ufo_error and recorded else "")
+        if ufo_error
+        else ("" if recorded else "UFO took no action that could be recorded as a step"),
+    }
+
+
+def run_record(req: Mapping[str, Any], events: Events, driver: Any, stop: threading.Event, recorder_loop: Optional[Callable[..., str]] = None) -> Dict[str, Any]:
+    """Recording by hand: the tester does the steps, their clicks and keys become workflow.yaml. No model."""
+    from .handrecord import HandRecorder
+
+    out = _out_dir(req)
+    desktop = req.get("desktop") or {}
+    app = str(req.get("app") or "")
+    spec = resolve_app(app, load_apps({"desktop": desktop}))
+    name = str(req.get("name") or f"Recorded in {app}")[:120]
+    names = _secret_names(desktop)
+    secrets = {n: os.environ[n] for n in names if os.environ.get(n)}
+    if recorder_loop is None:
+        from .handhooks import record_by_hand as recorder_loop
+    _check_desktop(driver)
+    recorder = HandRecorder(app=app, name=name, secrets=secrets, secret_names=names)
+    started = time.time()
+    reason = ""
+    error = ""
+    try:
+        reason = recorder_loop(driver, spec, recorder, stop, log=events.log, on_change=lambda e: events.emit({"type": "recorded", **e}))
+    except Exception as e:
+        error = f"Recording stopped with an error: {type(e).__name__}: {e}"[:2000]
+        events.log(error, "error")
+    finally:
+        # The text of a field still being typed into is read before the app closes.
+        workflow, skipped = recorder.finish()
+        try:
+            driver.close()
+        except Exception:
+            pass
+    recorded = len(workflow["steps"]) - 1
+    workflow_file = out / "workflow.yaml"
+    if recorded:
+        save(workflow, workflow_file)
+    checks = sum(1 for s in workflow["steps"] if s["op"] == "expect")
+    return {
+        "mode": "record",
+        "ok": recorded > 0 and not error,
+        "app": app,
+        "name": name,
+        "seconds": int(time.time() - started),
+        "recorded": recorded,
+        "checks": checks,
+        "stopped_by": reason or None,
+        "secrets": list(recorder.secrets_used),
+        "skipped": skipped,
+        "workflow": str(workflow_file) if recorded else None,
+        "steps": [{"index": i, "op": s["op"], "ok": True, "ms": 0, "target": s.get("target")} for i, s in enumerate(workflow["steps"], 1)],
+        "error": error or ("" if recorded else "nothing was recorded: no click or typing in the app's windows"),
+    }
+
+
+def _watch_stop(stream: IO[str], stop: threading.Event) -> None:
+    for line in stream:
+        if line.strip() == "stop":
+            stop.set()
+            return
+
+
+def serve(
+    stdin: IO[str] = sys.stdin,
+    stdout: IO[str] = sys.stdout,
+    driver_factory: Optional[Callable[[], Any]] = None,
+    runner: Optional[Callable[..., List[Dict[str, Any]]]] = None,
+    model_factory: Optional[Callable[..., Any]] = None,
+    recorder_loop: Optional[Callable[..., str]] = None,
+) -> int:
+    events = Events(stdout)
+    stop = threading.Event()
+    result: Dict[str, Any]
+    out: Optional[Path] = None
+    try:
+        line = stdin.readline()
+        req = json.loads(line or "{}")
+        if not isinstance(req, dict):
+            raise WorkflowError("the request must be a JSON object")
+        threading.Thread(target=_watch_stop, args=(stdin, stop), daemon=True).start()
+        out = _out_dir(req)
+        if driver_factory is None:
+            from .pywinauto_driver import PywinautoDriver as driver_factory
+        driver = driver_factory()
+        mode = req.get("mode")
+        if mode == "replay":
+            result = run_replay(req, events, driver, stop, model_factory)
+        elif mode == "explore":
+            result = run_explore(req, events, driver, runner, model_factory)
+        elif mode == "record":
+            result = run_record(req, events, driver, stop, recorder_loop)
+        else:
+            raise WorkflowError(f"mode must be replay, explore or record, not {mode!r}")
+    except Exception as e:
+        result = {"ok": False, "error": f"{type(e).__name__}: {e}" if not isinstance(e, (WorkflowError, ValueError)) else str(e)}
+    result.setdefault("format", FORMAT)
+    if out is not None:
+        (out / "run.json").write_text(json.dumps(result, ensure_ascii=False, indent=2, default=str), encoding="utf-8")
+    events.emit({"type": "done", "result": result})
+    return 0 if result.get("ok") else 1

package/py/forgeqe_desktop/session.py

@@ -0,0 +1,60 @@
+"""
+Whether this Windows session can be driven: replay and explore move the mouse and type into windows, which needs an
+interactive desktop that is unlocked. A Windows service (session 0), a locked screen, or a disconnected remote desktop
+session has none, and every step would time out; the run says why instead, before it starts the app.
+"""
+from __future__ import annotations
+
+import sys
+from typing import Any, Dict, Optional
+
+DESKTOP_SWITCHDESKTOP = 0x0100
+
+
+def desktop_state() -> Dict[str, Any]:
+    """``{"interactive", "unlocked", "session", "width", "height", "problem"}`` for this process's session."""
+    if sys.platform != "win32":
+        return {"interactive": False, "unlocked": False, "session": None, "width": 0, "height": 0, "problem": "not Windows"}
+    import ctypes
+    from ctypes import wintypes
+
+    kernel32 = ctypes.windll.kernel32
+    user32 = ctypes.windll.user32
+    session = wintypes.DWORD(0)
+    if not kernel32.ProcessIdToSessionId(kernel32.GetCurrentProcessId(), ctypes.byref(session)):
+        session.value = 0xFFFFFFFF
+    width, height = int(user32.GetSystemMetrics(0)), int(user32.GetSystemMetrics(1))
+    # The input desktop is the one that gets the keyboard and mouse; while the screen is locked it is the secure
+    # desktop, which this process may not open or switch to.
+    user32.OpenInputDesktop.restype = wintypes.HANDLE
+    handle = user32.OpenInputDesktop(0, False, DESKTOP_SWITCHDESKTOP)
+    unlocked = bool(handle) and bool(user32.SwitchDesktop(handle))
+    if handle:
+        user32.CloseDesktop(handle)
+    state: Dict[str, Any] = {"interactive": session.value != 0, "unlocked": unlocked, "session": int(session.value), "width": width, "height": height}
+    state["problem"] = problem_of(state)
+    return state
+
+
+def problem_of(state: Dict[str, Any]) -> Optional[str]:
+    if state.get("session") == 0:
+        return (
+            "this runs in session 0 (a Windows service), which has no desktop to click in. Run the CI runner or the "
+            "scheduled task in a logged-in user's session (docs/DESKTOP-CI.md)."
+        )
+    if not state.get("unlocked"):
+        return (
+            "the screen is locked, or this remote desktop session is disconnected, so no window takes input. Keep the "
+            "test machine's session logged in and unlocked (docs/DESKTOP-CI.md)."
+        )
+    if not state.get("width") or not state.get("height"):
+        return "this session has no screen (width or height is 0)."
+    return None
+
+
+def desktop_problem() -> Optional[str]:
+    """Why this session cannot be driven, or None; never raises."""
+    try:
+        return desktop_state().get("problem")
+    except Exception:
+        return None

package/py/forgeqe_desktop/ufo_bridge.py

@@ -0,0 +1,86 @@
+"""
+UFO² behind one adapter: its model calls go to the llms.json profile, and its config never holds a key.
+
+UFO picks a model client by each agent's ``API_TYPE`` through ``ufo.llm.base.BaseService.get_service``. ``install``
+makes the extra type ``forgeqe`` return ``ForgeQEService``; every other type still reaches UFO's own clients. UFO is
+pinned to an exact version and never edited; if a new version changes ``get_service``, ``install`` raises rather than
+running with UFO's own model config.
+"""
+from __future__ import annotations
+
+from typing import Any, Callable, Dict, List, Mapping, Optional, Tuple
+
+from .models import DesktopModel, require_vision
+
+API_TYPE = "forgeqe"
+AGENTS = ("HOST_AGENT", "APP_AGENT", "BACKUP_AGENT", "EVALUATION_AGENT")
+
+
+def agents_config(profile_name: str, profile: Mapping[str, Any]) -> Dict[str, Dict[str, Any]]:
+    """UFO's per-agent model blocks for a profile: the model's name only, no endpoint and no key."""
+    require_vision(profile_name, profile)
+    model = str(profile.get("deployment") or profile.get("model") or profile_name)
+    block = {
+        "VISUAL_MODE": True,
+        "REASONING_MODEL": False,
+        "API_TYPE": API_TYPE,
+        "API_BASE": "",
+        "API_KEY": "",
+        "API_VERSION": "",
+        "API_MODEL": model,
+        "FORGEQE_PROFILE": profile_name,
+    }
+    return {agent: dict(block) for agent in AGENTS}
+
+
+class ForgeQEService:
+    """UFO's model-client interface (``chat_completion``) on a ForgeQE ``DesktopModel``."""
+
+    def __init__(self, model: DesktopModel) -> None:
+        self.model = model
+
+    def chat_completion(
+        self,
+        messages: List[Mapping[str, Any]],
+        n: int = 1,
+        stream: bool = False,
+        temperature: Optional[float] = None,
+        max_tokens: Optional[int] = None,
+        top_p: Optional[float] = None,
+        **kwargs: Any,
+    ) -> Tuple[List[str], Optional[float]]:
+        # Cost is counted from tokens by ForgeQE's own ledger (on_call), so UFO's cost estimate is 0.
+        return [self.model.complete(messages, max_tokens=max_tokens) for _ in range(max(1, int(n or 1)))], 0.0
+
+
+def install(
+    profile_name: str,
+    profile: Mapping[str, Any],
+    on_call: Optional[Callable[[Dict[str, Any]], None]] = None,
+    base_service: Any = None,
+    model: Optional[DesktopModel] = None,
+) -> Callable[[], None]:
+    """
+    Route UFO's ``forgeqe`` API type to ``profile``. Returns a function that puts UFO's own ``get_service`` back.
+    ``base_service`` is UFO's ``BaseService`` (imported when not given).
+    """
+    require_vision(profile_name, profile)
+    if base_service is None:
+        from ufo.llm.base import BaseService as base_service  # type: ignore[no-redef]
+
+    original = getattr(base_service, "get_service", None)
+    if original is None or not callable(original):
+        raise RuntimeError("this UFO version has no BaseService.get_service; the desktop bridge needs updating")
+    service = ForgeQEService(model or DesktopModel(profile, on_call=on_call))
+
+    def get_service(name: Any, *args: Any, **kwargs: Any) -> Any:
+        if str(name).strip().lower() == API_TYPE:
+            return service
+        return original(name, *args, **kwargs)
+
+    base_service.get_service = staticmethod(get_service)
+
+    def restore() -> None:
+        base_service.get_service = staticmethod(original)
+
+    return restore