@capagents/forgeqe 1.3.1 → 2.0.0

package/src/desktop/report.ts

@@ -0,0 +1,155 @@
+import { writeFileSync } from "node:fs"
+import path from "node:path"
+
+import type { DesktopResult, DesktopStep } from "./bridge.ts"
+
+/**
+ * report.html for a desktop run: every value from run.json is escaped, since step errors, control names and goals come
+ * from the app under test and the model. Images are only the engine's own files: step screenshots (steps/NNN.png) and
+ * expect_image's copies (images/NNN-baseline|actual|diff.png).
+ */
+
+export function escapeHtml(value: unknown): string {
+  return String(value ?? "")
+    .replace(/&/g, "&")
+    .replace(/</g, "&lt;")
+    .replace(/>/g, "&gt;")
+    .replace(/"/g, "&quot;")
+    .replace(/'/g, "&#39;")
+    .replace(/\u2028/g, "&#8232;")
+    .replace(/\u2029/g, "&#8233;")
+}
+
+const SCREENSHOT = /^steps\/\d{3}\.png$/
+const STEP_IMAGE = /^images\/\d{3}-(baseline|actual|diff)\.png$/
+
+/** A step screenshot's relative path, or null for anything that is not one the engine wrote. */
+export function screenshotPath(value: unknown): string | null {
+  return typeof value === "string" && SCREENSHOT.test(value) ? value : null
+}
+
+/** An expect_image file's relative path (images/NNN-baseline|actual|diff.png), or null. */
+export function stepImagePath(value: unknown): string | null {
+  return typeof value === "string" && STEP_IMAGE.test(value) ? value : null
+}
+
+function imageCell(s: DesktopStep): string {
+  const im = s.images
+  if (!im) return ""
+  const pics = (["baseline", "actual", "diff"] as const)
+    .map((k) => {
+      const src = stepImagePath(im[k])
+      return src ? `<figure><a href="${src}"><img src="${src}" alt="${k} of step ${escapeHtml(s.index)}" loading="lazy"></a><figcaption>${k}</figcaption></figure>` : ""
+    })
+    .join("")
+  const share = typeof im.differs === "number" ? `<div class="meta">${escapeHtml((im.differs * 100).toFixed(2))}% of pixels differ</div>` : ""
+  return `<div class="images">${pics}</div>${share}`
+}
+
+function target(t: Record<string, unknown> | null | undefined): string {
+  if (!t) return ""
+  return Object.entries(t)
+    .filter(([k]) => k !== "path")
+    .map(([k, v]) => `${k}=${typeof v === "string" ? v : JSON.stringify(v)}`)
+    .join(", ")
+}
+
+function stepRow(s: DesktopStep): string {
+  const shot = screenshotPath(s.screenshot)
+  const healed = s.healed_target ? `<div class="heal">healed to ${escapeHtml(target(s.healed_target))}</div>` : ""
+  const tries = typeof s.attempts === "number" && s.attempts > 1 ? `<div class="heal">took ${escapeHtml(s.attempts)} tries</div>` : ""
+  const dismissed = (s.dismissed || []).length ? `<div class="heal">closed in the way: ${escapeHtml((s.dismissed || []).join(", "))}</div>` : ""
+  const note = s.note ? `<div class="meta">${escapeHtml(s.note)}</div>` : ""
+  return `<tr class="${s.ok ? "ok" : "fail"}">
+<td class="n">${escapeHtml(s.index)}</td>
+<td><span class="badge ${s.ok ? "ok" : "fail"}">${s.ok ? "passed" : "failed"}</span></td>
+<td><code>${escapeHtml(s.op)}</code></td>
+<td>${escapeHtml(target(s.target))}${healed}${tries}${dismissed}${note}${s.error ? `<div class="err">${escapeHtml(s.error)}</div>` : ""}${imageCell(s)}</td>
+<td class="n">${escapeHtml(s.ms)} ms</td>
+<td>${shot ? `<a href="${shot}"><img src="${shot}" alt="Screen after step ${escapeHtml(s.index)}" loading="lazy"></a>` : ""}</td>
+</tr>`
+}
+
+function flowsSection(r: DesktopResult): string {
+  if (!(r.flows || []).length) return ""
+  const rows = (r.flows || [])
+    .map((f) => `<tr class="${f.ok ? "ok" : "fail"}"><td><span class="badge ${f.ok ? "ok" : "fail"}">${f.ok ? "passed" : "failed"}</span></td><td>${escapeHtml(f.name)}${f.error ? `<div class="err">${escapeHtml(f.error)}</div>` : ""}</td></tr>`)
+    .join("")
+  return `<section><h2>Flows</h2><div class="table"><table><thead><tr><th>Result</th><th>Flow</th></tr></thead><tbody>${rows}</tbody></table></div></section>`
+}
+
+export function desktopReportHtml(r: DesktopResult, meta: { model: string; when: string; environment?: string; kind?: "Desktop" | "Mobile" }): string {
+  const kind = meta.kind || "Desktop"
+  const title = `${kind} ${r.mode === "explore" ? "explore" : r.mode === "record" ? "recording" : r.mode === "maestro" ? "Maestro flows" : "replay"}`
+  const status = r.ok ? "passed" : r.stopped ? "stopped" : "failed"
+  const summary =
+    r.mode === "maestro"
+      ? `${escapeHtml((r.flows || []).filter((f) => f.ok).length)} of ${escapeHtml((r.flows || []).length)} flows passed`
+      : r.mode === "record"
+      ? `${escapeHtml(r.recorded ?? 0)} steps recorded by hand, ${escapeHtml(r.checks ?? 0)} checks${r.secrets?.length ? `, types the secrets ${escapeHtml(r.secrets.join(", "))}` : ""}`
+      : r.mode === "explore"
+      ? `${escapeHtml(r.recorded ?? 0)} steps recorded, ${escapeHtml(r.checks ?? 0)} checks of the outcome${r.ufo_complete === false ? ", UFO judged the goal not done" : ""}`
+      : `${escapeHtml((r.steps || []).filter((s) => s.ok).length)} of ${escapeHtml(r.total ?? (r.steps || []).length)} steps passed`
+  const rows = (r.steps || []).map(stepRow).join("\n")
+  const heals = (r.heals || []).length
+    ? `<section><h2>Healed locators</h2><p>Proposed for review; the workflow file is unchanged until you accept them.</p><ul>${(r.heals || [])
+        .map((h) => `<li>Step ${escapeHtml(h.index)}: ${escapeHtml(target(h.old))} → ${escapeHtml(target(h.new))}</li>`)
+        .join("")}</ul></section>`
+    : ""
+  const skipped = (r.skipped || []).length
+    ? `<section><h2>Actions not recorded</h2><ul>${(r.skipped || []).map((s) => `<li>${escapeHtml(s.action)}: ${escapeHtml(s.reason)}</li>`).join("")}</ul></section>`
+    : ""
+  const refused = (r.refused || []).length
+    ? `<section><h2>Refused</h2><p>Actions ForgeQE did not let UFO take.</p><ul>${(r.refused || []).map((s) => `<li>${escapeHtml(s.action)} ${escapeHtml(JSON.stringify(s.args || {}))}</li>`).join("")}</ul></section>`
+    : ""
+  return `<!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>${escapeHtml(title)}: ${escapeHtml(r.name || r.goal || "")}</title>
+<style>
+:root { --bg: #f4f6f9; --surface: #ffffff; --line: #e3e8ee; --text: #1f2a44; --muted: #5f6b80; --brand: #0074bf;
+  --pass: #1b883c; --pass-bg: #e8f5ec; --fail: #c62828; --fail-bg: #fcecec; --warn: #c45c12; --radius: 14px; }
+@media (prefers-color-scheme: dark) { :root { --bg: #0f1424; --surface: #171e33; --line: #2a3350; --text: #e6eaf2; --muted: #9aa5bb;
+  --brand: #5bb6f0; --pass: #5fd07f; --pass-bg: #163322; --fail: #ff7b7b; --fail-bg: #3a1a1d; --warn: #f0a35e; } }
+* { box-sizing: border-box; }
+body { margin: 0; background: var(--bg); color: var(--text); font: 14px/1.5 system-ui, -apple-system, "Segoe UI", sans-serif; }
+main { max-width: 1100px; margin: 0 auto; padding: 24px 16px; }
+header, section { background: var(--surface); border: 1px solid var(--line); border-radius: var(--radius); padding: 16px 20px; margin-bottom: 16px; }
+h1 { font-size: 20px; margin: 0 0 4px; } h2 { font-size: 16px; margin: 0 0 8px; }
+.meta { color: var(--muted); }
+.badge { display: inline-block; padding: 1px 8px; border-radius: 999px; font-size: 12px; font-weight: 600; }
+.badge.ok { color: var(--pass); background: var(--pass-bg); } .badge.fail { color: var(--fail); background: var(--fail-bg); }
+.table { overflow-x: auto; }
+table { width: 100%; border-collapse: collapse; }
+th, td { text-align: left; vertical-align: top; padding: 8px; border-top: 1px solid var(--line); }
+th { color: var(--muted); font-weight: 600; border-top: 0; }
+td.n { white-space: nowrap; color: var(--muted); }
+.err { color: var(--fail); margin-top: 4px; white-space: pre-wrap; } .heal { color: var(--warn); margin-top: 4px; }
+img { max-width: 220px; border: 1px solid var(--line); border-radius: 6px; }
+.images { display: flex; flex-wrap: wrap; gap: 8px; margin-top: 6px; } figure { margin: 0; } figcaption { color: var(--muted); font-size: 12px; }
+code { font-family: ui-monospace, SFMono-Regular, Consolas, monospace; }
+</style>
+</head>
+<body>
+<main>
+<header>
+<h1>${escapeHtml(title)}: ${escapeHtml(r.name || r.goal || "")} <span class="badge ${r.ok ? "ok" : "fail"}">${status}</span></h1>
+<div class="meta">${summary} · app ${escapeHtml(r.app || "")}${meta.environment ? ` · environment ${escapeHtml(meta.environment)}` : ""} · model ${escapeHtml(meta.model)} · ${escapeHtml(meta.when)}</div>
+${r.error ? `<p class="err">${escapeHtml(r.error)}</p>` : ""}
+${r.workflow ? `<p class="meta">Workflow: <code>${escapeHtml(r.workflow)}</code>${r.driver ? ` · driver ${escapeHtml(r.driver)}` : ""}</p>` : ""}
+</header>
+${rows ? `<section><h2>Steps</h2><div class="table"><table><thead><tr><th>#</th><th>Result</th><th>Op</th><th>Control</th><th>Time</th><th>Screen</th></tr></thead><tbody>${rows}</tbody></table></div></section>` : ""}
+${flowsSection(r)}${heals}${skipped}${refused}
+</main>
+</body>
+</html>
+`
+}
+
+export function writeDesktopReport(outDir: string, r: DesktopResult, meta: { model: string; when?: string; environment?: string; kind?: "Desktop" | "Mobile" }): string {
+  const file = path.join(outDir, "report.html")
+  writeFileSync(file, desktopReportHtml(r, { model: meta.model, when: meta.when || new Date().toISOString(), ...(meta.environment ? { environment: meta.environment } : {}), ...(meta.kind ? { kind: meta.kind } : {}) }))
+  return file
+}

package/src/desktop/run.ts

@@ -0,0 +1,252 @@
+import { existsSync, mkdirSync, readFileSync, realpathSync, writeFileSync } from "node:fs"
+import path from "node:path"
+
+import YAML from "yaml"
+
+import { privateEnvNames } from "../childEnv.ts"
+import type { AppConfig } from "../config.ts"
+import type { LlmProfile } from "../llm.ts"
+import { runDesktop, type DesktopEvent, type DesktopRequest, type DesktopResult } from "./bridge.ts"
+import { writeDesktopReport } from "./report.ts"
+import { ensureDesktop, ufoDir } from "./setup.ts"
+
+/** Explore and replay as the CLI (`forgeqe desktop`) and ForgeQE's agent (desktop_* tools) both run them. */
+
+const VAR_NAME = /^[A-Za-z_][A-Za-z0-9_]{0,63}$/
+const VAR = /^([A-Za-z_][A-Za-z0-9_]{0,63})=(.*)$/s
+const MAX_VALUE = 10_000
+export const MAX_DATA_ROWS = 1000
+
+/** `name=value` words as workflow variables. */
+export function parseVariables(words: string[]): Record<string, string> {
+  const out: Record<string, string> = {}
+  for (const w of words) {
+    const m = VAR.exec(w)
+    if (!m) throw new Error(`"${w}" is not name=value; variables are letters, digits and _, e.g. customer=Contoso`)
+    out[m[1]!] = m[2]!
+  }
+  return out
+}
+
+/** Variables as a tool's caller gives them: names checked, values text, nothing else. */
+export function checkVariables(raw: unknown): Record<string, string> {
+  if (raw === undefined || raw === null) return {}
+  if (typeof raw !== "object" || Array.isArray(raw)) throw new Error("variables must be an object of name to text")
+  const out: Record<string, string> = {}
+  for (const [k, v] of Object.entries(raw as Record<string, unknown>)) {
+    if (!VAR_NAME.test(k)) throw new Error(`variable "${k}": names are letters, digits and _`)
+    if (typeof v !== "string" && typeof v !== "number") throw new Error(`variable ${k} must be text`)
+    const value = String(v)
+    if (value.length > MAX_VALUE) throw new Error(`variable ${k} is longer than ${MAX_VALUE} characters`)
+    out[k] = value
+  }
+  return out
+}
+
+/** A CSV file's rows as variables, one object per row, named by the header row. */
+export function parseCsv(text: string): Record<string, string>[] {
+  const rows: string[][] = []
+  let row: string[] = []
+  let field = ""
+  let quoted = false
+  const src = text.replace(/^/, "")
+  for (let i = 0; i < src.length; i++) {
+    const c = src[i]!
+    if (quoted) {
+      if (c === '"' && src[i + 1] === '"') {
+        field += '"'
+        i++
+      } else if (c === '"') quoted = false
+      else field += c
+      continue
+    }
+    if (c === '"' && field === "") quoted = true
+    else if (c === ",") {
+      row.push(field)
+      field = ""
+    } else if (c === "\n" || c === "\r") {
+      if (c === "\r" && src[i + 1] === "\n") i++
+      row.push(field)
+      field = ""
+      if (row.some((f) => f !== "")) rows.push(row)
+      row = []
+    } else field += c
+  }
+  if (quoted) throw new Error("the CSV file has an unclosed quote")
+  row.push(field)
+  if (row.some((f) => f !== "")) rows.push(row)
+  const [header, ...data] = rows
+  if (!header) throw new Error("the CSV file is empty")
+  const names = header.map((h) => h.trim())
+  for (const n of names) if (!VAR_NAME.test(n)) throw new Error(`CSV column "${n}": variable names are letters, digits and _`)
+  if (new Set(names).size !== names.length) throw new Error("the CSV header repeats a column")
+  if (data.length > MAX_DATA_ROWS) throw new Error(`the CSV file has ${data.length} rows; at most ${MAX_DATA_ROWS} run in one go`)
+  return data.map((r, i) => {
+    if (r.length !== names.length) throw new Error(`CSV row ${i + 2} has ${r.length} values; the header has ${names.length}`)
+    return Object.fromEntries(names.map((n, j) => [n, r[j]!]))
+  })
+}
+
+/** desktop.secrets that are ForgeQE's own (a model key, a tracker token): a workflow may never type those. */
+export function privateDesktopSecrets(cfg: AppConfig): string[] {
+  const own = new Set(privateEnvNames(cfg))
+  return cfg.desktop.secrets.filter((n) => own.has(n))
+}
+
+/** Why a profile cannot drive explore (it sends screenshots), or null. */
+export function visionProblem(name: string, p: LlmProfile): string | null {
+  const takes = typeof p.vision === "boolean" ? p.vision : p.provider !== "ollama"
+  return takes ? null : `llms.json profile "${name}" does not take images, and the desktop agent sends screenshots. Pick a vision model, or set "vision": true on the profile if it does take them.`
+}
+
+export function hasModel(cfg: AppConfig): boolean {
+  return Boolean(cfg.llm.model) && cfg.llm.provider !== "none" && cfg.llm.provider !== "mock"
+}
+
+/** Throws when the config cannot run desktop work: ForgeQE's own keys in desktop.secrets, or no model when one is needed. */
+export function checkDesktopConfig(cfg: AppConfig, needsModel: boolean): void {
+  const own = privateDesktopSecrets(cfg)
+  if (own.length) throw new Error(`desktop.secrets lists ${own.join(", ")}, which ForgeQE itself uses (a model key or a service token); a workflow may not type it.`)
+  if (needsModel && !hasModel(cfg)) throw new Error("No model is set up in llms.json. Run: forgeqe init, then forgeqe models (or replay with --no-heal).")
+}
+
+export function checkApp(cfg: AppConfig, app: string): void {
+  if (!Object.prototype.hasOwnProperty.call(cfg.desktop.apps, app)) {
+    throw new Error(`App "${app}" is not in forgeqe.yaml desktop.apps (configured: ${Object.keys(cfg.desktop.apps).join(", ") || "none"}).`)
+  }
+}
+
+function stamp(): string {
+  return new Date().toISOString().replace(/[:.]/g, "-").replace("T", "_").slice(0, 19)
+}
+
+export function desktopOutDir(cfg: AppConfig, kind: "explore" | "replay" | "record", out?: string): string {
+  const dir = out ? path.resolve(out) : path.join(cfg.export.outDir, `desktop-${kind}-${stamp()}`)
+  mkdirSync(dir, { recursive: true })
+  return dir
+}
+
+/**
+ * A workflow file a caller names, as its real path: a .yaml or .yml file inside one of `roots` (the working folder, the
+ * reports folder). A link that leads outside them is refused like a path that does.
+ */
+export function allowedWorkflow(file: string, roots: string[]): string {
+  if (!/\.ya?ml$/i.test(file)) throw new Error(`${file} is not a workflow (.yaml or .yml)`)
+  const resolved = path.resolve(roots[0] || process.cwd(), file)
+  if (!existsSync(resolved)) throw new Error(`No workflow at ${resolved}`)
+  const real = realpathSync(resolved)
+  for (const root of roots) {
+    if (!existsSync(root)) continue
+    const rel = path.relative(realpathSync(root), real)
+    if (rel && !rel.startsWith("..") && !path.isAbsolute(rel)) return real
+  }
+  throw new Error(`${file} is outside the working folder and the reports folder; workflows are read from those only.`)
+}
+
+/** Write accepted heals into the workflow, keeping its comments and layout. */
+export function applyHeals(file: string, heals: NonNullable<DesktopResult["heals"]>): void {
+  const doc = YAML.parseDocument(readFileSync(file, "utf8"))
+  for (const h of heals) doc.setIn(["steps", h.index - 1, "target"], doc.createNode(h.new))
+  writeFileSync(file, doc.toString())
+}
+
+export type DesktopRun = { result: DesktopResult; report: string; outDir: string }
+
+function modelLabel(cfg: AppConfig): string {
+  return `${cfg.activeProfile} (${cfg.llm.provider}/${cfg.llm.deployment || cfg.llm.model})`
+}
+
+async function finish(cfg: AppConfig, outDir: string, result: DesktopResult): Promise<DesktopRun> {
+  const report = writeDesktopReport(outDir, result, { model: modelLabel(cfg), ...(cfg.environments?.active ? { environment: cfg.environments.active } : {}) })
+  if (result.heals?.length) writeFileSync(path.join(outDir, "heals.json"), JSON.stringify(result.heals, null, 2))
+  return { result, report, outDir }
+}
+
+/** Replay one workflow; heals are returned and saved to heals.json, never written into the workflow here. */
+export async function replayWorkflow(opts: {
+  cfg: AppConfig
+  workflow: string
+  variables?: Record<string, string>
+  heal: boolean
+  updateBaselines?: boolean
+  outDir: string
+  signal?: AbortSignal
+  onEvent?: (e: DesktopEvent) => void
+  log?: (text: string) => void
+}): Promise<DesktopRun> {
+  const python = await ensureDesktop(opts.log || (() => {}))
+  const request: DesktopRequest = {
+    mode: "replay",
+    outDir: opts.outDir,
+    llm: opts.cfg.llm,
+    profileName: opts.cfg.activeProfile,
+    desktop: opts.cfg.desktop,
+    workflow: opts.workflow,
+    variables: opts.variables || {},
+    heal: opts.heal && hasModel(opts.cfg),
+    ...(opts.updateBaselines ? { updateBaselines: true } : {}),
+  }
+  const result = await runDesktop({ python, request, signal: opts.signal, onEvent: opts.onEvent })
+  return finish(opts.cfg, opts.outDir, result)
+}
+
+/** Explore a goal with UFO² and save what it did, with checks of the outcome, as workflow.yaml. */
+export async function exploreGoal(opts: {
+  cfg: AppConfig
+  goal: string
+  app: string
+  name?: string
+  outDir: string
+  signal?: AbortSignal
+  onEvent?: (e: DesktopEvent) => void
+  log?: (text: string) => void
+}): Promise<DesktopRun> {
+  const python = await ensureDesktop(opts.log || (() => {}), { ufo: true })
+  const request: DesktopRequest = {
+    mode: "explore",
+    outDir: opts.outDir,
+    llm: opts.cfg.llm,
+    profileName: opts.cfg.activeProfile,
+    desktop: opts.cfg.desktop,
+    ufoDir: ufoDir(),
+    goal: opts.goal,
+    app: opts.app,
+    name: opts.name,
+  }
+  const result = await runDesktop({ python, request, signal: opts.signal, onEvent: opts.onEvent })
+  return finish(opts.cfg, opts.outDir, result)
+}
+
+/** Record by hand: start the app, and save what the tester does in it (until the hotkey, or the app closes) as workflow.yaml. */
+export async function recordByHand(opts: {
+  cfg: AppConfig
+  app: string
+  name?: string
+  outDir: string
+  signal?: AbortSignal
+  onEvent?: (e: DesktopEvent) => void
+  log?: (text: string) => void
+}): Promise<DesktopRun> {
+  const python = await ensureDesktop(opts.log || (() => {}))
+  const request: DesktopRequest = {
+    mode: "record",
+    outDir: opts.outDir,
+    llm: opts.cfg.llm,
+    profileName: opts.cfg.activeProfile,
+    desktop: opts.cfg.desktop,
+    app: opts.app,
+    ...(opts.name ? { name: opts.name } : {}),
+  }
+  // A stop ends the recording; the engine keeps what was recorded, so it gets time to save it.
+  const result = await runDesktop({ python, request, signal: opts.signal, onEvent: opts.onEvent, graceMs: 60_000 })
+  return finish(opts.cfg, opts.outDir, result)
+}
+
+/** What a finished recording or explore wrote, and what still has to be set before it replays. */
+export function recordingNotes(cfg: AppConfig, result: DesktopResult): string[] {
+  const notes: string[] = []
+  const missing = (result.secrets || []).filter((n) => !cfg.desktop.secrets.includes(n))
+  if (missing.length) notes.push(`The workflow types ${missing.join(", ")} from password fields: set ${missing.length === 1 ? "it" : "them"} as environment variables and add ${missing.length === 1 ? "it" : "them"} to desktop.secrets (or the environment's secrets in env.yaml).`)
+  if (result.workflow && (result.checks ?? 0) === 0) notes.push(`No checks: replay proves only that the steps run. Record again with Ctrl+Shift+F11 over a control that shows the outcome, or add expect steps.`)
+  return notes
+}

package/src/desktop/setup.ts

@@ -0,0 +1,134 @@
+import { spawnSync } from "node:child_process"
+import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"
+import path from "node:path"
+
+import { engineEnv, ensureUv, run, uvEnv } from "../engine/setup.ts"
+import { forgeqeHome, type Log } from "../tools.ts"
+
+/**
+ * The desktop engine's own Python environment (~/.forgeqe/desktop/.venv), separate from the browser engine's: UFO pins
+ * its libraries (openai, pydantic) at versions the browser engine's library does not run with. Windows only.
+ */
+
+/** The UFO commit the desktop engine is tested with; also in py/forgeqe_desktop/ufo_runner.py (a test checks they match). */
+export const UFO_COMMIT = "eae3c0a8d168c47cb10df0652e293e092eda1078"
+export const UFO_REPO = "https://github.com/microsoft/UFO.git"
+
+/**
+ * The UFO commit to set up: UFO_COMMIT, or FORGEQE_UFO_COMMIT when it is a full 40-character commit id, which is how
+ * the UFO update check (.github/workflows/forgeqe-desktop-ufo-update.yml) tries a newer UFO before it is pinned.
+ * Anything shorter or other than hex is refused, so the version set up is always exact.
+ */
+export function ufoCommit(env: NodeJS.ProcessEnv = process.env): string {
+  const raw = (env.FORGEQE_UFO_COMMIT || "").trim()
+  if (!raw) return UFO_COMMIT
+  if (!/^[0-9a-f]{40}$/.test(raw)) throw new Error("FORGEQE_UFO_COMMIT must be a full 40-character UFO commit id (lowercase hex).")
+  return raw
+}
+
+/** UFO's requirements pin pandas 1.4.3, which has Windows wheels up to Python 3.10 only. */
+export const DESKTOP_PYTHON = "3.10"
+
+/** Replay's libraries, at the versions UFO's requirements.txt pins, so one environment runs both. */
+export const DESKTOP_PACKAGES = ["pywinauto==0.6.8", "PyYAML==6.0.1", "openai==1.66.2", "azure-identity==1.16.1", "Pillow==11.3.0", "comtypes==1.2.0"]
+
+export const NOT_WINDOWS = "forgeqe desktop drives Windows apps, so explore, replay and setup run on Windows. validate and apps work anywhere."
+
+export function desktopHome(): string {
+  return path.join(forgeqeHome(), "desktop")
+}
+
+function venvDir(): string {
+  return path.join(desktopHome(), ".venv")
+}
+
+export function desktopPython(platform: string = process.platform): string {
+  return platform === "win32" ? path.join(venvDir(), "Scripts", "python.exe") : path.join(venvDir(), "bin", "python")
+}
+
+export function ufoDir(): string {
+  return path.join(desktopHome(), `ufo-${ufoCommit().slice(0, 12)}`)
+}
+
+function markerFile(): string {
+  return path.join(desktopHome(), "installed.json")
+}
+
+type Marker = { packages?: string[]; ufo?: string; python?: string }
+
+function marker(): Marker {
+  try {
+    return JSON.parse(readFileSync(markerFile(), "utf8")) as Marker
+  } catch {
+    return {}
+  }
+}
+
+export type DesktopDoctor = { python: string; ready: boolean; ufo: boolean; error?: string }
+
+/** Whether the environment imports what replay needs, and whether the pinned UFO checkout is in place. */
+export function desktopDoctor(python: string = desktopPython()): DesktopDoctor {
+  if (!existsSync(python)) return { python, ready: false, ufo: false, error: "the desktop engine is not set up" }
+  const r = spawnSync(python, ["-c", "import pywinauto, yaml, openai, forgeqe_desktop.service"], { env: engineEnv(), encoding: "utf8" })
+  const ready = !r.error && r.status === 0
+  const m = marker()
+  const ufo = m.ufo === ufoCommit() && existsSync(path.join(ufoDir(), "ufo", "__init__.py"))
+  return { python, ready, ufo, ...(ready ? {} : { error: (r.stderr || r.error?.message || "").trim().split(/\r?\n/).slice(-1)[0] || "imports failed" }) }
+}
+
+function gitHead(dir: string): string {
+  const r = spawnSync("git", ["-C", dir, "rev-parse", "HEAD"], { encoding: "utf8" })
+  return r.status === 0 ? r.stdout.trim() : ""
+}
+
+/** Clone UFO and check out exactly ufoCommit(); the commit id is the content's hash, so a matching HEAD is the pinned code. */
+async function ensureUfo(log: Log): Promise<string> {
+  const commit = ufoCommit()
+  const dir = ufoDir()
+  if (existsSync(dir) && gitHead(dir) === commit) return dir
+  const git = spawnSync("git", ["--version"], { encoding: "utf8" })
+  if (git.error || git.status !== 0) throw new Error("Setting up UFO needs git on PATH (https://git-scm.com/download/win).")
+  rmSync(dir, { recursive: true, force: true })
+  log(`Fetching UFO ${commit.slice(0, 12)}${commit === UFO_COMMIT ? "" : " (FORGEQE_UFO_COMMIT, not the pinned one)"} into ${dir}.`)
+  await run("git", ["clone", "--filter=blob:none", "--no-checkout", UFO_REPO, dir], log)
+  await run("git", ["-C", dir, "checkout", "--detach", commit], log)
+  const head = gitHead(dir)
+  if (head !== commit) {
+    rmSync(dir, { recursive: true, force: true })
+    throw new Error(`UFO checkout is at ${head || "nothing"}, not ${commit}; removed it.`)
+  }
+  return dir
+}
+
+/**
+ * Set up the desktop engine: its environment with replay's libraries, and with `ufo` the pinned UFO checkout and its
+ * requirements (large: UFO's retrieval needs PyTorch). Returns the environment's Python.
+ */
+export async function ensureDesktop(log: Log, opts: { ufo?: boolean; force?: boolean; platform?: string } = {}): Promise<string> {
+  if ((opts.platform || process.platform) !== "win32") throw new Error(NOT_WINDOWS)
+  const python = desktopPython()
+  const m: Marker = opts.force ? {} : marker()
+  const packagesCurrent = JSON.stringify(m.packages || []) === JSON.stringify(DESKTOP_PACKAGES) && m.python === DESKTOP_PYTHON
+  if (!opts.force && packagesCurrent && existsSync(python) && (!opts.ufo || m.ufo === ufoCommit()) && desktopDoctor(python).ready) return python
+
+  // An environment on another Python (an earlier ForgeQE used 3.11) is rebuilt rather than mixed.
+  if (opts.force || (existsSync(python) && m.python !== DESKTOP_PYTHON)) rmSync(venvDir(), { recursive: true, force: true })
+  mkdirSync(desktopHome(), { recursive: true })
+  const uv = await ensureUv(log)
+  if (!uv) throw new Error("The desktop engine needs uv to set up its Python; uv could not be downloaded. Set HTTPS_PROXY if you are behind a proxy.")
+  log(`Setting up the desktop engine in ${venvDir()}.`)
+  if (!existsSync(python)) await run(uv, ["venv", "--python", DESKTOP_PYTHON, venvDir()], log, uvEnv(uv))
+  const extra: string[] = []
+  let ufo: string | undefined
+  if (opts.ufo) {
+    const dir = await ensureUfo(log)
+    extra.push("-r", path.join(dir, "requirements.txt"))
+    ufo = ufoCommit()
+  }
+  await run(uv, ["pip", "install", "--python", python, ...DESKTOP_PACKAGES, ...extra], log, uvEnv(uv), "uv pip install <desktop packages>")
+  const kept = m.python === DESKTOP_PYTHON ? m.ufo : undefined
+  writeFileSync(markerFile(), JSON.stringify({ packages: DESKTOP_PACKAGES, python: DESKTOP_PYTHON, ...(ufo || kept ? { ufo: ufo || kept } : {}) }, null, 2))
+  const doc = desktopDoctor(python)
+  if (!doc.ready) throw new Error(`The desktop engine was installed but does not start: ${doc.error}`)
+  return python
+}

package/src/engine/bridge.ts

@@ -11,6 +11,8 @@ import type { BrowserProvider, BrowserRemoteConfig, LoginConfig } from "../confi
 import { debugLogging, logEvent } from "../sessionLog.ts"
 import { engineEnv } from "./setup.ts"
 import { AZURE_IDENTITY_ENV, entraPayload } from "../entra.ts"
+import { ghAuthToken } from "../cases/github.ts"
+import { engineRoute } from "../copilot.ts"
 
 export type EngineRequest = {
   goal: string
@@ -152,7 +154,7 @@ function toMemoryInfo(m: EngineMemory | undefined): MemoryInfo | undefined {
   }
 }
 
-function llmPayload(p: LlmProfile): Record<string, unknown> {
+export function llmPayload(p: LlmProfile): Record<string, unknown> {
   const out: Record<string, unknown> = { provider: p.provider, model: p.model }
   if (p.endpoint) out.endpoint = p.endpoint
   if (p.deployment) out.deployment = p.deployment
@@ -161,6 +163,8 @@ function llmPayload(p: LlmProfile): Record<string, unknown> {
   if (p.auth === "entra") Object.assign(out, entraPayload(p))
   else if (p.apiKeyEnv) out.api_key_env = p.apiKeyEnv
   if (typeof p.temperature === "number") out.temperature = p.temperature
+  if (typeof p.vision === "boolean") out.vision = p.vision
+  if (typeof p.reasoning === "boolean") out.reasoning = p.reasoning
   return out
 }
 
@@ -240,12 +244,15 @@ export async function runEngine(opts: {
   // Each model call goes to the usage ledger under this run (its folder's name) and case; what the per-call events
   // missed (an engine that sends only a total) is recorded as one unattributed call when the run ends.
   const scope = { runId: path.basename(req.outDir), ...(req.caseId ? { caseId: req.caseId } : {}) }
-  const model = req.llm.deployment || req.llm.model
+  // A github_copilot profile's engine runs on GitHub Models; its token goes only into the engine's environment.
+  const route = engineRoute(req.llm, ghAuthToken)
+  const llm = route.profile
+  const model = llm.deployment || llm.model
   const callTotal: Usage = { input: 0, output: 0 }
   const [bin, ...args] = opts.command || [opts.python, "-m", "forgeqe_engine", "run"]
   const child = spawn(bin!, args, {
-    env: engineEnv(debugLogging() ? { ...req.env, BROWSER_USE_LOGGING_LEVEL: "debug" } : req.env, [
-      ...(req.llm.auth === "entra" ? AZURE_IDENTITY_ENV : req.llm.apiKeyEnv ? [req.llm.apiKeyEnv] : []),
+    env: engineEnv({ ...req.env, ...(debugLogging() ? { BROWSER_USE_LOGGING_LEVEL: "debug" } : {}), ...route.env }, [
+      ...(llm.auth === "entra" ? AZURE_IDENTITY_ENV : llm.apiKeyEnv ? [llm.apiKeyEnv] : []),
       ...browserProviderEnvNames(req.browser || { provider: "local" }),
       ...loginEnvNames(req.logins || []),
     ]),
@@ -266,7 +273,7 @@ export async function runEngine(opts: {
   const payload = {
     goal: req.goal,
     out_dir: req.outDir,
-    llm: llmPayload(req.llm),
+    llm: llmPayload(llm),
     start_url: req.startUrl || null,
     headed: req.headed,
     max_steps: req.maxSteps,

package/src/engine/setup.ts

@@ -68,6 +68,9 @@ export function engineEnv(extra?: Record<string, string>, allow: string[] = []):
   const env = allowListEnv(process.env, allow)
   env.PYTHONPATH = [ENGINE_SOURCE, env.PYTHONPATH].filter(Boolean).join(path.delimiter)
   env.PYTHONUNBUFFERED = "1"
+  // Requests, prompts and logs are UTF-8; Windows would otherwise read pipes and files in its ANSI code page.
+  env.PYTHONUTF8 = "1"
+  env.PYTHONIOENCODING = "utf-8"
   return { ...env, ...extra }
 }
 
@@ -78,7 +81,7 @@ function managedUv(): string {
 }
 
 /** A uv ForgeQE downloaded keeps its Pythons under ~/.forgeqe too, so deleting that folder removes everything. */
-function uvEnv(uv: string): Record<string, string> {
+export function uvEnv(uv: string): Record<string, string> {
   return uv === managedUv() ? { UV_PYTHON_INSTALL_DIR: path.join(toolsDir(), "python") } : {}
 }
 
@@ -103,7 +106,7 @@ export function uvAsset(platform: string = process.platform, arch: string = proc
 }
 
 /** System uv, else a uv ForgeQE downloaded earlier, else download it now. Null only when the download fails. */
-async function ensureUv(log: Log): Promise<string | null> {
+export async function ensureUv(log: Log): Promise<string | null> {
   const found = findUv()
   if (found) return found
   try {
@@ -142,7 +145,7 @@ function findSystemPython(): string | null {
   return null
 }
 
-function run(cmd: string, args: string[], log: Log, env: Record<string, string> = {}, shown?: string): Promise<void> {
+export function run(cmd: string, args: string[], log: Log, env: Record<string, string> = {}, shown?: string): Promise<void> {
   log(`$ ${shown || [path.basename(cmd), ...args].join(" ")}`)
   return new Promise((resolve, reject) => {
     const child = spawn(cmd, args, {