@capagents/forgeqe 1.3.2 → 2.0.1

package/py/forgeqe_desktop/ufo_runner.py

@@ -0,0 +1,305 @@
+"""
+Explore: UFO² does a goal on the live app, in this process, with ForgeQE's model, rules and trace.
+
+UFO is a pinned checkout that ForgeQE owns (``forgeqe desktop setup --ufo``); its source is never edited. Each run:
+
+- writes UFO's ``config/ufo/agents.yaml`` (the ``forgeqe`` model type, no key) and ``mcp.yaml`` (UI tools only: no
+  ``CommandLineExecutor``, which runs shell commands, and no COM or PDF executors);
+- starts the app from ``desktop.apps`` itself, so UFO never launches a program;
+- wraps UFO's ``ActionExecutor.execute`` to record each action with the control's UIA properties (UFO's own log has the
+  control's name and a per-step id, not its AutomationId), and to refuse keys that reach outside the app;
+- answers UFO's console questions (its safe guard for sensitive actions) with no, since nobody is at the console.
+"""
+from __future__ import annotations
+
+import builtins
+import io
+import os
+import re
+import shutil
+import sys
+from contextlib import contextmanager
+from pathlib import Path
+from typing import Any, Callable, Dict, Iterator, List, Mapping, Optional
+
+import yaml
+
+from .models import DesktopModel
+from .ufo_bridge import agents_config, install
+
+# The UFO commit ForgeQE is tested with; setup checks out exactly this one.
+UFO_COMMIT = "eae3c0a8d168c47cb10df0652e293e092eda1078"
+UFO_REPO = "https://github.com/microsoft/UFO.git"
+
+MCP_CONFIG: Dict[str, Any] = {
+    "HostAgent": {
+        "default": {
+            "data_collection": [{"namespace": "UICollector", "type": "local", "start_args": [], "reset": False}],
+            "action": [{"namespace": "HostUIExecutor", "type": "local", "start_args": [], "reset": False}],
+        }
+    },
+    "AppAgent": {
+        "default": {
+            "data_collection": [{"namespace": "UICollector", "type": "local", "start_args": [], "reset": False}],
+            "action": [{"namespace": "AppUIExecutor", "type": "local", "start_args": [], "reset": False}],
+        }
+    },
+}
+
+# UFO's own key syntax for the Windows and context-menu keys, which reach programs outside the app.
+_REFUSED_UFO_KEYS = re.compile(r"\{\s*(VK_)?(L|R)?WIN\b|\{\s*(VK_)?APPS\b", re.I)
+_TRACE_FUNCTIONS = ("click_input", "set_edit_text", "keyboard_input", "select", "click_on_coordinates", "drag_on_coordinates", "wheel_mouse_input", "texts", "summary")
+# The UI actions UFO's AppUIExecutor runs at the pinned commit; anything else (a shell or COM command a later UFO might
+# route through ActionExecutor) is refused rather than run.
+ALLOWED_FUNCTIONS = frozenset(_TRACE_FUNCTIONS)
+
+
+class ExploreError(RuntimeError):
+    """Explore cannot run here; the message says what to set up."""
+
+
+# pywinauto's send_keys syntax: ^ % + hold Ctrl, Alt, Shift for the next key or (group); {NAME}, {NAME n},
+# {NAME down} / {NAME up}; ~ is Enter.
+_MOD_CHARS = {"^": "ctrl", "%": "alt", "+": "shift"}
+_HELD_NAMES = {
+    "vk_control": "ctrl", "vk_lcontrol": "ctrl", "vk_rcontrol": "ctrl", "control": "ctrl", "ctrl": "ctrl",
+    "vk_menu": "alt", "vk_lmenu": "alt", "vk_rmenu": "alt", "menu": "alt", "alt": "alt",
+    "vk_shift": "shift", "vk_lshift": "shift", "vk_rshift": "shift", "shift": "shift",
+}
+_ESC = {"esc", "escape", "vk_escape"}
+_TAB = {"tab", "vk_tab"}
+_DEL = {"del", "delete", "vk_delete"}
+
+
+def _key_presses(keys: str):
+    """Each key pressed by a send_keys string, with the modifiers held at that moment."""
+    held = set()  # held with {NAME down}
+    group: List[set] = []  # modifiers held for a (group)
+    pending: set = set()  # ^ % + waiting for their key
+    i = 0
+    while i < len(keys):
+        c = keys[i]
+        if c in _MOD_CHARS:
+            pending.add(_MOD_CHARS[c])
+            i += 1
+            continue
+        if c == "(":
+            group.append(set(pending))
+            pending = set()
+            i += 1
+            continue
+        if c == ")":
+            if group:
+                group.pop()
+            i += 1
+            continue
+        mods = held | pending | set().union(*group) if group else held | pending
+        if c == "{":
+            end = keys.find("}", i + 2)  # "{}}" types a brace
+            if end < 0:
+                end = len(keys)
+            words = keys[i + 1:end].strip().lower().split()
+            i = end + 1
+            pending = set()
+            if not words:
+                continue
+            name = words[0]
+            if len(words) > 1 and words[1] in ("down", "up") and name in _HELD_NAMES:
+                (held.add if words[1] == "down" else held.discard)(_HELD_NAMES[name])
+                continue
+            yield name, mods
+            continue
+        yield ("enter" if c == "~" else c.lower()), mods
+        pending = set()
+        i += 1
+
+
+def refused_keys(keys: str) -> bool:
+    """
+    Keys that reach outside the app: the Windows and menu keys, and the chords Windows itself handles (Ctrl+Esc and
+    Ctrl+Shift+Esc open the Start menu and Task Manager, Alt+Tab and Alt+Esc switch to other programs, Ctrl+Alt+Del).
+    """
+    text = keys or ""
+    if _REFUSED_UFO_KEYS.search(text):
+        return True
+    for name, mods in _key_presses(text):
+        if name in _ESC and "ctrl" in mods:
+            return True
+        if (name in _ESC or name in _TAB) and "alt" in mods:
+            return True
+        if name in _DEL and {"ctrl", "alt"} <= mods:
+            return True
+    return False
+
+
+def write_ufo_config(ufo_dir: Path, profile_name: str, profile: Mapping[str, Any]) -> None:
+    """UFO's agents.yaml for the profile (the template's other settings kept) and an mcp.yaml with UI tools only."""
+    cfg_dir = ufo_dir / "config" / "ufo"
+    if not cfg_dir.is_dir():
+        raise ExploreError(f"{ufo_dir} is not a UFO checkout (no config/ufo). Run: forgeqe desktop setup --ufo")
+    template = cfg_dir / "agents.yaml.template"
+    base: Dict[str, Any] = {}
+    if template.is_file():
+        base = yaml.safe_load(template.read_text(encoding="utf-8")) or {}
+    base.update(agents_config(profile_name, profile))
+    (cfg_dir / "agents.yaml").write_text(yaml.safe_dump(base, sort_keys=False), encoding="utf-8")
+    (cfg_dir / "mcp.yaml").write_text(yaml.safe_dump(MCP_CONFIG, sort_keys=False), encoding="utf-8")
+
+
+def _control_props(control: Any) -> Dict[str, str]:
+    info = getattr(control, "element_info", None)
+    out: Dict[str, str] = {}
+    for key, attr in (("automation_id", "automation_id"), ("name", "name"), ("control_type", "control_type"), ("class_name", "class_name")):
+        try:
+            value = getattr(info, attr, None)
+        except Exception:
+            value = None
+        if isinstance(value, str) and value:
+            out[key] = value
+    return out
+
+
+def install_trace(action_executor: Any, trace: List[Dict[str, Any]]) -> Callable[[], None]:
+    """
+    Record every action UFO executes into ``trace`` (the format ``record.steps_from_trace`` reads) and refuse keys that
+    reach outside the app. Returns a function that puts UFO's ``execute`` back.
+    """
+    original = action_executor.execute
+
+    def execute(self: Any, action: Any, puppeteer: Any, control_dict: Dict[str, Any], application_window: Any = None) -> Any:
+        function = str(getattr(action, "function", "") or "")
+        arguments = dict(getattr(action, "arguments", {}) or {})
+        target = getattr(action, "target", None)
+        control = (control_dict or {}).get(getattr(target, "id", None)) if target is not None else None
+        entry: Dict[str, Any] = {"action": function, "args": arguments, "control": _control_props(control) if control is not None else {}}
+        if function not in ALLOWED_FUNCTIONS:
+            entry["ok"] = False
+            entry["refused"] = "not a UI action"
+            trace.append(entry)
+            raise ValueError(f"ForgeQE refused {function!r}: explore runs UI actions only.")
+        if function == "keyboard_input" and refused_keys(str(arguments.get("keys") or "")):
+            entry["ok"] = False
+            entry["refused"] = "keys that reach outside the app"
+            trace.append(entry)
+            raise ValueError("ForgeQE refused this key sequence: the Windows and menu keys reach programs outside the app.")
+        try:
+            result = original(self, action, puppeteer, control_dict, application_window)
+        except Exception as e:
+            entry["ok"] = False
+            entry["error"] = str(e)[:500]
+            trace.append(entry)
+            raise
+        entry["ok"] = True
+        if function in _TRACE_FUNCTIONS:
+            trace.append(entry)
+        return result
+
+    action_executor.execute = execute
+
+    def restore() -> None:
+        action_executor.execute = original
+
+    return restore
+
+
+@contextmanager
+def _no_console_answers(log: Callable[[str], None]) -> Iterator[None]:
+    original = builtins.input
+
+    def refuse(prompt: str = "") -> str:
+        log(f"UFO asked a question at the console and was answered no: {str(prompt)[:200]}")
+        return "N"
+
+    builtins.input = refuse
+    try:
+        yield
+    finally:
+        builtins.input = original
+
+
+@contextmanager
+def _console_to_stderr() -> Iterator[None]:
+    """
+    UFO prints its panels (with emoji) to stdout, which is the engine's JSON event stream; send them to stderr instead,
+    and never fail on a character the console's code page lacks (Windows' cp1252 has no emoji).
+    """
+    old_out = sys.stdout
+    err = sys.stderr
+    try:
+        stream: Any = io.TextIOWrapper(err.buffer, encoding=err.encoding or "utf-8", errors="replace", line_buffering=True, write_through=True)
+    except (AttributeError, ValueError):
+        stream = err
+    sys.stdout = stream
+    try:
+        yield
+    finally:
+        try:
+            stream.flush()
+            if stream is not err:
+                stream.detach()
+        except Exception:
+            pass
+        sys.stdout = old_out
+
+
+@contextmanager
+def _in_ufo(ufo_dir: Path, argv: List[str]) -> Iterator[None]:
+    old_cwd, old_argv, old_path = os.getcwd(), list(sys.argv), list(sys.path)
+    os.chdir(ufo_dir)
+    sys.argv = argv
+    sys.path.insert(0, str(ufo_dir))
+    try:
+        yield
+    finally:
+        os.chdir(old_cwd)
+        sys.argv = old_argv
+        sys.path[:] = old_path
+
+
+def safe_task_name(name: str) -> str:
+    return re.sub(r"[^A-Za-z0-9_-]+", "-", name).strip("-")[:60] or "forgeqe"
+
+
+def run_ufo(
+    ufo_dir: Path,
+    goal: str,
+    task: str,
+    profile_name: str,
+    profile: Mapping[str, Any],
+    out_dir: Path,
+    on_call: Optional[Callable[[Dict[str, Any]], None]] = None,
+    log: Callable[[str], None] = lambda s: None,
+    model: Optional[DesktopModel] = None,
+    trace: Optional[List[Dict[str, Any]]] = None,
+) -> List[Dict[str, Any]]:
+    """
+    Run UFO on ``goal`` and return the trace; UFO's own logs for the task are copied into ``out_dir/ufo``. Actions are
+    appended to ``trace`` as they happen, so a caller keeps what UFO did even when UFO later raises.
+    """
+    if sys.platform != "win32":
+        raise ExploreError("explore drives Windows apps and runs only on Windows.")
+    if not (ufo_dir / "ufo" / "__init__.py").is_file():
+        raise ExploreError(f"UFO is not set up in {ufo_dir}. Run: forgeqe desktop setup --ufo")
+    write_ufo_config(ufo_dir, profile_name, profile)
+    task = safe_task_name(task)
+    trace = [] if trace is None else trace
+    with _in_ufo(ufo_dir, ["ufo", "--task", task, "--request", goal, "--log-level", "WARNING"]):
+        from ufo.automator.action_execution import ActionExecutor  # type: ignore
+        from ufo.llm.base import BaseService  # type: ignore
+
+        restore_model = install(profile_name, profile, on_call=on_call, base_service=BaseService, model=model)
+        restore_trace = install_trace(ActionExecutor, trace)
+        try:
+            with _no_console_answers(log), _console_to_stderr():
+                import asyncio
+
+                from ufo import ufo as ufo_main  # type: ignore  # parses sys.argv when imported
+
+                asyncio.run(ufo_main.main())
+        finally:
+            restore_trace()
+            restore_model()
+            logs = ufo_dir / "logs" / task
+            if logs.is_dir():
+                shutil.copytree(logs, out_dir / "ufo", dirs_exist_ok=True)
+    return trace

package/py/forgeqe_desktop/visual.py

@@ -0,0 +1,112 @@
+"""
+Visual checks (``expect_image``): a control or window as captured now, compared with a baseline image kept next to
+the workflow, in ``<workflow name>.baselines/``.
+
+Two pixels count as different when any colour channel differs by more than ``PIXEL_THRESHOLD`` (anti-aliasing and
+ClearType shift colours a little between runs); the step fails when the share of different pixels, outside the
+``ignore`` regions (a clock, a date), is more than its ``tolerance``. A first run with no baseline saves one and
+passes, so a new check is recorded on a machine the team trusts and compared on every run after.
+
+Baseline names are checked file names (``workflow.check_image_name``) and their real path must stay inside the
+baselines folder, so a workflow cannot read or overwrite another file through a name or a link.
+"""
+from __future__ import annotations
+
+import io
+import os
+import shutil
+from pathlib import Path
+from typing import Any, Dict, Mapping, Optional, Sequence, Tuple
+
+from .workflow import WorkflowError, check_image_name
+
+PIXEL_THRESHOLD = 32
+DEFAULT_TOLERANCE = 0.01
+
+
+def baselines_dir(workflow_file: str | os.PathLike) -> Path:
+    p = Path(workflow_file)
+    return p.with_name(f"{p.stem}.baselines")
+
+
+def baseline_path(folder: Path, name: str) -> Path:
+    """``folder/name`` for a checked image name, refusing a path or a link that leads out of ``folder``."""
+    check_image_name("expect_image", name)
+    folder.mkdir(parents=True, exist_ok=True)
+    root = Path(os.path.realpath(folder))
+    if root.parent != Path(os.path.realpath(folder.parent)):
+        raise WorkflowError(f"expect_image: {folder} is a link to another folder; baselines stay next to the workflow")
+    path = folder / name
+    real = Path(os.path.realpath(path))
+    if real.parent != root:
+        raise WorkflowError(f"expect_image: {name} leads outside {folder}")
+    return path
+
+
+def compare(baseline: bytes, actual: bytes, ignore: Sequence[Sequence[int]] = (), threshold: int = PIXEL_THRESHOLD) -> Tuple[Optional[float], Optional[bytes], str]:
+    """``(share of pixels that differ, diff PNG, problem)``; the share is None when the sizes differ."""
+    from PIL import Image, ImageChops
+
+    a = Image.open(io.BytesIO(baseline)).convert("RGB")
+    b = Image.open(io.BytesIO(actual)).convert("RGB")
+    if a.size != b.size:
+        return None, None, f"the image is {b.size[0]}x{b.size[1]} now and {a.size[0]}x{a.size[1]} in the baseline (another screen scale or window size?)"
+    r, g, bl = ImageChops.difference(a, b).split()
+    worst = ImageChops.lighter(ImageChops.lighter(r, g), bl)
+    mask = worst.point(lambda v: 255 if v > threshold else 0)
+    counted = a.size[0] * a.size[1]
+    if ignore:
+        blank = Image.new("L", a.size, 0)
+        keep = Image.new("L", a.size, 255)
+        for x, y, w, h in ignore:
+            box = (x, y, min(x + w, a.size[0]), min(y + h, a.size[1]))
+            if box[0] < box[2] and box[1] < box[3]:
+                keep.paste(blank.crop(box), box)
+        mask = ImageChops.multiply(mask, keep)
+        counted = max(1, keep.histogram()[255])
+    differing = mask.histogram()[255]
+    share = differing / max(1, counted)
+    diff = None
+    if differing:
+        faded = Image.blend(b, Image.new("RGB", b.size, (255, 255, 255)), 0.6)
+        red = Image.new("RGB", b.size, (220, 0, 0))
+        out = io.BytesIO()
+        Image.composite(red, faded, mask).save(out, format="PNG")
+        diff = out.getvalue()
+    return share, diff, ""
+
+
+def image_checker(workflow_file: str | os.PathLike, out_dir: Path, update: bool = False):
+    """The ``compare_image`` callback for replay: baselines next to the workflow, this run's images in ``out_dir/images``."""
+    folder = baselines_dir(workflow_file)
+    images = out_dir / "images"
+
+    def check(index: int, step: Mapping[str, Any], png: bytes) -> Tuple[bool, str, Dict[str, Any]]:
+        name = step["image"]
+        path = baseline_path(folder, name)
+        images.mkdir(parents=True, exist_ok=True)
+        actual = f"images/{index:03d}-actual.png"
+        (out_dir / actual).write_bytes(png)
+        files: Dict[str, Any] = {"name": name, "actual": actual}
+        if update or not path.is_file():
+            had = path.is_file()
+            path.write_bytes(png)
+            files["baseline_saved"] = True
+            return True, f"{'updated' if had else 'saved'} the baseline {name}; later runs compare against it", files
+        baseline = f"images/{index:03d}-baseline.png"
+        shutil.copyfile(path, out_dir / baseline)
+        files["baseline"] = baseline
+        tolerance = float(step.get("tolerance", DEFAULT_TOLERANCE))
+        share, diff, problem = compare(path.read_bytes(), png, step.get("ignore") or [])
+        if share is None:
+            return False, f"{name}: {problem}", files
+        files["differs"] = round(share, 5)
+        if diff:
+            files["diff"] = f"images/{index:03d}-diff.png"
+            (out_dir / files["diff"]).write_bytes(diff)
+        if share > tolerance:
+            return False, f"{name}: {share:.2%} of the pixels differ from the baseline (at most {tolerance:.2%} may)", files
+        return True, f"{name}: matches the baseline ({share:.2%} of the pixels differ)", files
+
+    return check
+