@capagents/forgeqe 0.0.0-stage → 1.0.0

package/src/code/replay.ts

@@ -0,0 +1,772 @@
+import { spawn } from "node:child_process"
+import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"
+import path from "node:path"
+
+import YAML from "yaml"
+
+import { withoutNames } from "../childEnv.ts"
+import { withoutTelemetry } from "../privacy.ts"
+import { redactSecrets, sessionLoginEnv } from "../sessionLogins.ts"
+import { toolsDir, type Log } from "../tools.ts"
+import type { BrowserProfile, Check, CheckResult, Locator, Step } from "./handoff.ts"
+
+/**
+ * The replay check: before Scribe starts, ForgeQE runs the browser agent's recorded steps as a Playwright script,
+ * so Scribe learns which steps work in the library its test uses, and where one fails, what the page showed there.
+ */
+
+export const PLAYWRIGHT_VERSION = "1.63.0"
+const STEP_TIMEOUT_MS = 10_000
+const NAVIGATION_TIMEOUT_MS = 30_000
+const PASS_TIMEOUT_MS = 180_000
+/** The replay stops after this many failed steps in a row; the steps after them depend on those. */
+const MAX_MISSES = 3
+const MAX_ALTERNATIVES = 4
+
+export type Setup = "playwright" | "forgeqe"
+
+export type ReplayStep = {
+  n: number
+  action: string
+  ok: boolean
+  /** Not replayed (an action with no Playwright equivalent), or not run because the steps before it failed. */
+  skipped?: boolean
+  error?: string
+  ms?: number
+  url?: string
+  screenshot?: string
+  snapshot?: string
+  /** On a failure: how many elements each of the step's other recorded locators matched at that moment. */
+  alternatives?: { locator: string; matches: number }[]
+  /** On a failure: console errors, failed requests, and HTTP errors while the step ran. */
+  events?: string[]
+}
+
+export type ReplayPass = {
+  setup: Setup
+  steps: ReplayStep[]
+  webdriver: boolean | null
+  finalUrl: string
+  error: string
+  /** Every replayed step worked and the last page is the run's final page. */
+  ok: boolean
+  /** The last page has the same host and path as the run's final page. */
+  reached: boolean
+  ms: number
+  /** Playwright's page snapshot of each page the replay visited, as the replay left it, in visiting order. */
+  pages?: { url: string; snapshot: string }[]
+  /** The browser agent's checks, looked up on the page the replay ended on. */
+  checks?: CheckResult[]
+}
+
+export type Replay = {
+  dir: string
+  /** The replayed steps as a Playwright script: the reference Scribe ports into the repository's framework. */
+  script: string
+  steps: number
+  passes: ReplayPass[]
+  /** The replay could not run at all (no steps, Playwright could not be installed, the browser did not start). */
+  error?: string
+}
+
+export const SETUP_LABEL: Record<Setup, string> = {
+  playwright: "Playwright's default launch",
+  forgeqe: "without Playwright's automation switches (--enable-automation left out, --disable-blink-features=AutomationControlled added; navigator.webdriver is false, as in ForgeQE's browser)",
+}
+
+const q = (s: unknown) => JSON.stringify(String(s ?? ""))
+
+/** A Playwright locator expression for one recorded locator, built from its type and value. */
+export function locatorExpr(l: Locator): string | null {
+  const v = l.value || ""
+  if (!v) return null
+  switch (l.type) {
+    case "test_id":
+      // An attribute selector, not getByTestId: a repository can point getByTestId at another attribute.
+      return `page.locator(${q(`[${l.attr || "data-testid"}=${JSON.stringify(v)}]`)})`
+    case "id":
+      return `page.locator(${q(l.css || `[id=${JSON.stringify(v)}]`)})`
+    case "role": {
+      const i = v.indexOf("|")
+      const role = i < 0 ? v : v.slice(0, i)
+      const name = i < 0 ? "" : v.slice(i + 1)
+      return name ? `page.getByRole(${q(role)}, { name: ${q(name)}, exact: true })` : `page.getByRole(${q(role)})`
+    }
+    case "name":
+    case "href":
+      return l.css ? `page.locator(${q(l.css)})` : null
+    case "label":
+      return `page.getByLabel(${q(v)}, { exact: true })`
+    case "placeholder":
+      return `page.getByPlaceholder(${q(v)}, { exact: true })`
+    case "alt":
+      return `page.getByAltText(${q(v)}, { exact: true })`
+    case "title":
+      return `page.getByTitle(${q(v)}, { exact: true })`
+    case "text":
+      return `page.getByText(${q(v)}, { exact: true })`
+    case "css":
+      return `page.locator(${q(v)})`
+    case "xpath":
+      return `page.locator(${q(`xpath=${v}`)})`
+    default:
+      return null
+  }
+}
+
+function describe(step: Step): string {
+  const target = step.label || step.locator?.name || "element"
+  switch (step.op) {
+    case "navigate":
+      return `navigate ${step.url}`
+    case "click":
+      return `click ${q(target)}`
+    case "input":
+      return `fill ${q(target)} with ${step.totp_env ? `a one-time code from ${step.totp_env}` : step.text_env ? `process.env.${step.text_env}` : q(step.text)}`
+    case "select":
+      return `select ${q(step.text)} in ${q(target)}`
+    case "press":
+      return `press ${step.keys}`
+    case "wait":
+      return `wait ${step.seconds || 1}s`
+    case "scroll":
+      return `scroll ${step.down === false ? "up" : "down"} ${step.pages || 1} page(s)`
+    case "back":
+      return "go back"
+    case "evaluate":
+      return "run the agent's JavaScript"
+    default:
+      return `${step.op || "unknown"}${step.action_keys?.length ? ` (${step.action_keys.join(", ")})` : ""}`
+  }
+}
+
+/** The body of one step in the script, or null when it has no Playwright equivalent. */
+function stepCode(step: Step): string | null {
+  const loc = step.locators?.map(locatorExpr).find(Boolean) || null
+  switch (step.op) {
+    case "navigate":
+      return step.url ? `(page) => page.goto(${q(step.url)})` : null
+    case "click":
+      return loc ? `(page) => ${loc}.click()` : null
+    case "input": {
+      if (!loc) return null
+      const value = step.totp_env ? `totp(${q(step.totp_env)})` : step.text_env ? `env(${q(step.text_env)})` : q(step.text)
+      return [
+        "async (page) => {",
+        `    const field = ${loc}`,
+        ...(step.clear ? ['    await field.fill("")'] : []),
+        `    await field.pressSequentially(${value}, { delay: 40 })`,
+        "  }",
+      ].join("\n")
+    }
+    case "select":
+      return loc ? `(page) => ${loc}.selectOption(${q(step.text)})` : null
+    case "press":
+      return step.keys ? `(page) => page.keyboard.press(${q(step.keys)})` : null
+    case "wait":
+      return `(page) => page.waitForTimeout(${Math.round(Number(step.seconds || 1) * 1000)})`
+    case "scroll":
+      return `(page) => page.mouse.wheel(0, ${step.down === false ? -1 : 1} * ${Number(step.pages || 1)} * (page.viewportSize()?.height || 800))`
+    case "back":
+      return "(page) => page.goBack()"
+    case "evaluate":
+      return step.code ? `(page) => page.evaluate(${q(step.code)})` : null
+    default:
+      return null
+  }
+}
+
+/** The replay script (the reference for Scribe) and, apart, each step's other locators for diagnosing a failure. */
+export function replayScript(steps: Step[]): { script: string; alternatives: string } {
+  const lines = [
+    "// ForgeQE's replay of the browser agent's steps in Playwright: a reference for the test, not the test.",
+    "// step(n, action, run) runs one step on the current page; env(NAME) reads a secret, totp(NAME) makes a one-time code.",
+    "export default async function flow(step, { env, totp }) {",
+  ]
+  const alternatives: string[] = ["export default {"]
+  steps.forEach((s, i) => {
+    const n = i + 1
+    const code = stepCode(s)
+    lines.push(code ? `  await step(${n}, ${q(describe(s))}, ${code})` : `  await step(${n}, ${q(`not replayed: ${describe(s)}`)}, null)`)
+    const others = (s.locators || []).slice(1, 1 + MAX_ALTERNATIVES).flatMap((l) => {
+      const expr = locatorExpr(l)
+      return expr ? [`[${q(`${l.type}: ${l.value}`)}, (page) => ${expr}]`] : []
+    })
+    if (others.length) alternatives.push(`  ${n}: [${others.join(", ")}],`)
+  })
+  lines.push("}")
+  alternatives.push("}")
+  return { script: `${lines.join("\n")}\n`, alternatives: `${alternatives.join("\n")}\n` }
+}
+
+/** Launch and context options for one setup, from the run's browser. */
+export function replayOptions(b: BrowserProfile, setup: Setup): { launch: Record<string, unknown>; context: Record<string, unknown> } {
+  const launch: Record<string, unknown> = { headless: Boolean(b.headless) }
+  if (b.channel) launch.channel = b.channel
+  else if (b.executable && existsSync(b.executable)) launch.executablePath = b.executable
+  else launch.channel = "chrome"
+  if (setup === "forgeqe") {
+    launch.ignoreDefaultArgs = ["--enable-automation"]
+    launch.args = ["--disable-blink-features=AutomationControlled"]
+  }
+  const context: Record<string, unknown> = {}
+  if (b.viewport?.length === 2) context.viewport = { width: b.viewport[0], height: b.viewport[1] }
+  if (b.language) context.locale = b.language
+  if (b.timezone) context.timezoneId = b.timezone
+  if (b.user_agent) context.userAgent = b.user_agent
+  if (b.extra_http_headers && Object.keys(b.extra_http_headers).length) context.extraHTTPHeaders = b.extra_http_headers
+  return { launch, context }
+}
+
+/** Runs flow.mjs in one browser setup and writes what each step did to the result file. */
+function runnerSource(playwright: string): string {
+  return `import { createHmac } from "node:crypto"
+import { readFileSync, writeFileSync } from "node:fs"
+import path from "node:path"
+import { chromium } from ${JSON.stringify(playwright)}
+
+const config = JSON.parse(readFileSync(process.argv[2], "utf8"))
+const { default: flow } = await import(config.flow)
+const { default: alternatives } = await import(config.alternatives)
+const out = { steps: [], webdriver: null, finalUrl: "", error: "" }
+const save = () => writeFileSync(config.result, JSON.stringify(out, null, 2))
+const PROGRESS = /^(attempting .* action|retrying .* action|scrolling into view if needed|done scrolling|element is visible, enabled and stable|waiting for element to be visible, enabled and stable|waiting \\d+ms)$/
+const firstLines = (e) => {
+  const seen = new Set()
+  for (const raw of String(e?.message || e).replace(/\\u001b\\[[0-9;]*m/g, "").split("\\n")) {
+    const line = raw.trim().replace(/^\\d+ × /, "").replace(/^- /, "").slice(0, 240)
+    if (line && line !== "Call log:" && !PROGRESS.test(line)) seen.add(line)
+  }
+  return [...seen].slice(0, 6).join(" | ")
+}
+
+const env = (name) => {
+  const v = process.env[name]
+  if (v === undefined || v === "") throw new Error(name + " is not set in the environment")
+  return v
+}
+const totp = (name) => {
+  const alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567"
+  const bits = [...env(name).replace(/[\\s=]/g, "").toUpperCase()].map((c) => alphabet.indexOf(c).toString(2).padStart(5, "0")).join("")
+  const key = Buffer.from((bits.match(/.{8}/g) || []).map((b) => parseInt(b, 2)))
+  const counter = Buffer.alloc(8)
+  counter.writeBigUInt64BE(BigInt(Math.floor(Date.now() / 30000)))
+  const h = createHmac("sha1", key).update(counter).digest()
+  return String((h.readUInt32BE(h[19] & 15) & 0x7fffffff) % 1000000).padStart(6, "0")
+}
+
+const q = JSON.stringify
+const norm = (s) => String(s || "").replace(/\\s+/g, " ").trim().toLowerCase()
+// The nearest container a test can scope to: a test id, a stable id, a label, or a table, list, or section with a class.
+const container = (el) => {
+  const wraps = new Set(["TABLE", "SECTION", "ASIDE", "NAV", "FORM", "DIALOG", "HEADER", "FOOTER", "MAIN", "ARTICLE", "UL", "OL"])
+  for (let e = el.parentElement; e && e !== document.body; e = e.parentElement) {
+    const testId = e.getAttribute("data-testid")
+    if (testId) return '[data-testid="' + testId + '"]'
+    if (e.id && /^[A-Za-z][\\w-]*$/.test(e.id) && !/\\d{3,}/.test(e.id)) return "#" + e.id
+    const label = e.getAttribute("aria-label")
+    if (label) return e.tagName.toLowerCase() + '[aria-label="' + label + '"]'
+    if (wraps.has(e.tagName) && e.classList.length) return e.tagName.toLowerCase() + "." + e.classList[0]
+  }
+  return ""
+}
+// The part of the page a heading names: an element labelled by it, the section or region it heads, or else the
+// element right after it (or after the wrapper that holds only the heading), with how many links and list items it has.
+// It runs in the page, so it uses nothing from this script.
+const sectionOf = (h) => {
+  const q = JSON.stringify
+  const stable = (e) => {
+    const testId = e.getAttribute("data-testid")
+    if (testId) return 'page.locator(' + q('[data-testid=' + q(testId) + ']') + ')'
+    if (e.id && /^[A-Za-z][\\w-]*$/.test(e.id) && !/\\d{3,}/.test(e.id)) return 'page.locator(' + q("#" + e.id) + ')'
+    const label = e.getAttribute("aria-label")
+    if (label) return 'page.locator(' + q(e.tagName.toLowerCase() + '[aria-label="' + label + '"]') + ')'
+    return ""
+  }
+  let s = h.id ? document.querySelector('[aria-labelledby~="' + CSS.escape(h.id) + '"]') : null
+  let how = "labelled by the heading"
+  if (!s) {
+    const box = h.closest("section, article, [role=region]")
+    if (box && box.querySelector("h1, h2, h3, h4, h5, h6, [role=heading]") === h) { s = box; how = "the " + box.tagName.toLowerCase() + " it heads" }
+  }
+  if (!s) {
+    let head = h
+    while (head.parentElement && head.parentElement !== document.body && head.parentElement.children.length === 1) head = head.parentElement
+    s = head.nextElementSibling
+    how = "the element right after the heading"
+  }
+  if (!s) return null
+  const visible = (e) => !!(e.offsetWidth || e.offsetHeight || e.getClientRects().length)
+  return {
+    locator: stable(s),
+    how,
+    tag: s.tagName.toLowerCase(),
+    links: [...s.querySelectorAll("a[href]")].filter(visible).length,
+    items: [...s.querySelectorAll("li")].filter(visible).length,
+  }
+}
+// A check's text on the page: the first of these locators with a visible match, else a form field holding it.
+const findCheck = async (page, c) => {
+  const t = c.text
+  const tries = [
+    ["page.getByRole(\\"heading\\", { name: " + q(t) + ", exact: true })", () => page.getByRole("heading", { name: t, exact: true })],
+    ["page.getByText(" + q(t) + ", { exact: true })", () => page.getByText(t, { exact: true })],
+    ["page.getByRole(\\"heading\\", { name: " + q(t) + " })", () => page.getByRole("heading", { name: t })],
+    ["page.getByText(" + q(t) + ")", () => page.getByText(t)],
+  ]
+  for (const [locator, make] of tries) {
+    const loc = make()
+    const matches = await loc.count().catch(() => 0)
+    const where = []
+    let shown = null
+    for (let i = 0; i < Math.min(matches, 3); i++) {
+      const el = loc.nth(i)
+      if (!(await el.isVisible().catch(() => false))) continue
+      shown = shown || el
+      const tag = await el.evaluate((e) => e.tagName.toLowerCase()).catch(() => "")
+      const inside = await el.evaluate(container).catch(() => "")
+      where.push(tag + (inside ? " in " + inside : ""))
+    }
+    if (!where.length) continue
+    const section = locator.includes("heading") ? await shown.evaluate(sectionOf).catch(() => null) : null
+    return { ...c, locator, matches, where, ...(section ? { section } : {}) }
+  }
+  const fields = page.locator("input:visible, textarea:visible, select:visible")
+  const count = Math.min(await fields.count().catch(() => 0), 60)
+  for (let i = 0; i < count; i++) {
+    const f = fields.nth(i)
+    if (!norm(await f.inputValue().catch(() => "")).includes(norm(t))) continue
+    const d = await f.evaluate((e) => ({ id: e.id, name: e.getAttribute("name"), label: e.getAttribute("aria-label"), testId: e.getAttribute("data-testid"), placeholder: e.getAttribute("placeholder") })).catch(() => ({}))
+    const locator = d.testId ? "page.locator(" + q("[data-testid=" + q(d.testId) + "]") + ")"
+      : d.id ? "page.locator(" + q(/^[A-Za-z][\\w-]*$/.test(d.id) ? "#" + d.id : "[id=" + q(d.id) + "]") + ")"
+      : d.label ? "page.getByLabel(" + q(d.label) + ")"
+      : d.name ? "page.locator(" + q("[name=" + q(d.name) + "]") + ")"
+      : d.placeholder ? "page.getByPlaceholder(" + q(d.placeholder) + ")" : ""
+    if (locator) return { ...c, locator, matches: 1, field: true }
+  }
+  return { ...c, matches: 0 }
+}
+
+let browser
+try {
+  browser = await chromium.launch(config.launch)
+  const context = await browser.newContext(config.context)
+  context.setDefaultTimeout(config.stepTimeoutMs)
+  context.setDefaultNavigationTimeout(config.navigationTimeoutMs)
+  const events = []
+  const note = (text) => { if (!events.includes(text)) events.push(text) }
+  const bare = (u) => { try { const x = new URL(u); return x.origin + x.pathname } catch { return u } }
+  const watch = (p) => {
+    p.on("console", (m) => { if (m.type() === "error") note("console error: " + m.text().slice(0, 200)) })
+    p.on("requestfailed", (r) => note("request failed: " + r.method() + " " + bare(r.url()).slice(0, 120) + " (" + (r.failure()?.errorText || "") + ")"))
+    p.on("response", (r) => { if (r.status() >= 400) note("HTTP " + r.status() + ": " + r.request().method() + " " + bare(r.url()).slice(0, 120)) })
+  }
+  let page = await context.newPage()
+  watch(page)
+  context.on("page", (p) => { page = p; watch(p) })
+  out.webdriver = await page.evaluate(() => navigator.webdriver).catch(() => null)
+  const pages = new Map()
+  const snap = () => page.locator("body").ariaSnapshot({ timeout: 5000 }).catch(() => "")
+  // A page can change its URL at once and re-render a moment later: wait until two snapshots a second apart agree.
+  const settled = async () => {
+    let last = await snap()
+    for (let i = 0; i < 6; i++) {
+      await page.waitForTimeout(1000)
+      const next = await snap()
+      if (next && next === last) break
+      last = next
+    }
+    return last
+  }
+  const keepPage = async (settle = false) => {
+    let key = ""
+    try { const u = new URL(page.url()); if (u.protocol.startsWith("http")) key = u.host + u.pathname } catch {}
+    if (!key) return
+    const aria = settle ? await settled() : await snap()
+    if (aria) pages.set(key, aria)
+  }
+  const writePages = () => {
+    out.pages = [...pages].map(([url, aria], i) => {
+      const snapshot = path.join(config.outDir, config.setup + "-page-" + (i + 1) + ".aria.yml")
+      writeFileSync(snapshot, aria)
+      return { url, snapshot }
+    })
+  }
+  let misses = 0
+  const step = async (n, action, run) => {
+    if (!run) return void out.steps.push({ n, action, ok: true, skipped: true })
+    if (misses >= config.maxMisses) return void out.steps.push({ n, action, ok: false, skipped: true, error: "not run: the steps before it failed" })
+    const started = Date.now()
+    events.length = 0
+    const rec = { n, action, ok: true }
+    try {
+      await run(page)
+      await page.waitForLoadState("domcontentloaded", { timeout: 5000 }).catch(() => {})
+      misses = 0
+      await keepPage()
+    } catch (e) {
+      misses++
+      rec.ok = false
+      rec.error = firstLines(e)
+      const shot = path.join(config.outDir, config.setup + "-step-" + n + ".png")
+      if (await page.screenshot({ path: shot }).then(() => true, () => false)) rec.screenshot = shot
+      const aria = await page.locator("body").ariaSnapshot({ timeout: 5000 }).catch(() => "")
+      if (aria) {
+        rec.snapshot = path.join(config.outDir, config.setup + "-step-" + n + ".aria.yml")
+        writeFileSync(rec.snapshot, aria)
+      }
+      rec.alternatives = []
+      for (const [locator, make] of alternatives[n] || []) rec.alternatives.push({ locator, matches: await make(page).count().catch(() => -1) })
+      rec.events = events.slice(0, 6)
+    }
+    rec.ms = Date.now() - started
+    rec.url = page.url()
+    out.steps.push(rec)
+    save()
+  }
+  await flow(step, { env, totp })
+  if (config.finalPage) await page.waitForURL((u) => u.host + u.pathname === config.finalPage, { timeout: config.stepTimeoutMs }).catch(() => {})
+  await keepPage(true)
+  out.finalUrl = page.url()
+  writePages()
+  if (config.checks.length) {
+    out.checks = []
+    for (const c of config.checks) out.checks.push(await findCheck(page, c))
+  }
+} catch (e) {
+  out.error = firstLines(e)
+} finally {
+  save()
+  await browser?.close().catch(() => {})
+}
+`
+}
+
+function playwrightDir(): string {
+  return path.join(toolsDir(), `playwright-core-${PLAYWRIGHT_VERSION}`)
+}
+
+/** playwright-core's entry point, installed with Bun into ~/.forgeqe/tools on first use. It drives the installed Chrome. */
+export async function ensurePlaywright(log: Log, bun: string = process.execPath): Promise<string> {
+  const dir = playwrightDir()
+  const entry = path.join(dir, "node_modules", "playwright-core", "index.mjs")
+  if (existsSync(entry)) return entry
+  log(`Installing Playwright ${PLAYWRIGHT_VERSION} for the replay check (one time).`)
+  mkdirSync(dir, { recursive: true })
+  writeFileSync(path.join(dir, "package.json"), `${JSON.stringify({ name: "forgeqe-playwright", private: true })}\n`)
+  const r = await new Promise<{ code: number | null; out: string }>((resolve) => {
+    const child = spawn(bun, ["add", `playwright-core@${PLAYWRIGHT_VERSION}`], { cwd: dir, env: withoutTelemetry(process.env), stdio: ["ignore", "pipe", "pipe"] })
+    let out = ""
+    child.stdout.on("data", (b) => (out += b))
+    child.stderr.on("data", (b) => (out += b))
+    child.on("error", (e) => resolve({ code: -1, out: e.message }))
+    child.on("close", (code) => resolve({ code, out }))
+  })
+  if (r.code !== 0 || !existsSync(entry)) {
+    rmSync(dir, { recursive: true, force: true })
+    throw new Error(`Could not install Playwright ${PLAYWRIGHT_VERSION}: ${r.out.trim().split("\n").slice(-3).join(" ")}`)
+  }
+  for (const old of readdirSync(toolsDir())) {
+    if (old.startsWith("playwright-core-") && old !== path.basename(dir)) rmSync(path.join(toolsDir(), old), { recursive: true, force: true })
+  }
+  return entry
+}
+
+function hostPath(url: string): string {
+  try {
+    const u = new URL(url)
+    return `${u.host}${u.pathname}`
+  } catch {
+    return url.split("?")[0]
+  }
+}
+
+function readJson<T>(file: string): T | null {
+  try {
+    return JSON.parse(readFileSync(file, "utf8")) as T
+  } catch {
+    return null
+  }
+}
+
+async function runPass(opts: {
+  dir: string
+  runner: string
+  setup: Setup
+  browser: BrowserProfile
+  finalUrl: string
+  checks: Check[]
+  signal?: AbortSignal
+  timeoutMs: number
+  hide: string[]
+}): Promise<ReplayPass> {
+  const started = Date.now()
+  const result = path.join(opts.dir, `${opts.setup}.json`)
+  rmSync(result, { force: true })
+  const configFile = path.join(opts.dir, `${opts.setup}.config.json`)
+  writeFileSync(
+    configFile,
+    JSON.stringify({
+      ...replayOptions(opts.browser, opts.setup),
+      setup: opts.setup,
+      flow: path.join(opts.dir, "flow.mjs"),
+      alternatives: path.join(opts.dir, "alternatives.mjs"),
+      result,
+      outDir: opts.dir,
+      finalPage: opts.finalUrl ? hostPath(opts.finalUrl) : "",
+      stepTimeoutMs: STEP_TIMEOUT_MS,
+      navigationTimeoutMs: NAVIGATION_TIMEOUT_MS,
+      maxMisses: MAX_MISSES,
+      checks: opts.checks,
+    }),
+  )
+  const timedOut = await new Promise<boolean>((resolve) => {
+    const child = spawn(process.execPath, [opts.runner, configFile], {
+      cwd: opts.dir,
+      env: { ...withoutNames(process.env, opts.hide), ...sessionLoginEnv(), FORCE_COLOR: "0", NO_COLOR: "1" },
+      stdio: "ignore",
+    })
+    let late = false
+    const stop = () => child.kill("SIGKILL")
+    const timer = setTimeout(() => {
+      late = true
+      stop()
+    }, opts.timeoutMs)
+    opts.signal?.addEventListener("abort", stop, { once: true })
+    child.on("error", () => resolve(false))
+    child.on("close", () => {
+      clearTimeout(timer)
+      opts.signal?.removeEventListener("abort", stop)
+      resolve(late)
+    })
+  })
+  const raw = readJson<{ steps: ReplayStep[]; webdriver: boolean | null; finalUrl: string; error: string; pages?: ReplayPass["pages"]; checks?: CheckResult[] }>(result)
+  const out = raw || { steps: [], webdriver: null, finalUrl: "", error: "the replay wrote no result" }
+  if (timedOut) out.error = `stopped after ${Math.round(opts.timeoutMs / 1000)}s`
+  const steps = JSON.parse(redactSecrets(JSON.stringify(out.steps))) as ReplayStep[]
+  const pages = (out.pages || []).filter((p) => existsSync(p.snapshot))
+  for (const file of [...pages.map((p) => p.snapshot), ...steps.flatMap((s) => (s.snapshot && existsSync(s.snapshot) ? [s.snapshot] : []))]) {
+    writeFileSync(file, redactSecrets(readFileSync(file, "utf8")))
+  }
+  const reached = Boolean(out.finalUrl) && (!opts.finalUrl || hostPath(out.finalUrl) === hostPath(opts.finalUrl))
+  return {
+    setup: opts.setup,
+    steps,
+    webdriver: out.webdriver,
+    finalUrl: out.finalUrl,
+    error: redactSecrets(out.error || ""),
+    reached,
+    ok: !out.error && !timedOut && reached && steps.every((s) => s.ok),
+    ms: Date.now() - started,
+    ...(pages.length ? { pages } : {}),
+    ...(out.checks?.length ? { checks: JSON.parse(redactSecrets(JSON.stringify(out.checks))) as CheckResult[] } : {}),
+  }
+}
+
+function workflow(runDir: string): { steps?: Step[]; checks?: Check[] } {
+  const file = path.join(runDir, "workflow.yaml")
+  return existsSync(file) ? (YAML.parse(readFileSync(file, "utf8")) as { steps?: Step[]; checks?: Check[] }) || {} : {}
+}
+
+function recordedSteps(runDir: string): Step[] {
+  return workflow(runDir).steps || []
+}
+
+/** The results the browser agent recorded as checked, with the text the page showed for each. */
+export function recordedChecks(runDir: string): Check[] {
+  try {
+    return (workflow(runDir).checks || []).filter((c) => c && typeof c.text === "string" && c.text.trim())
+  } catch {
+    return []
+  }
+}
+
+/** The run recorded something to replay beyond opening a page, or checks to look up on the page it ended on. */
+export function canReplay(runDir: string): boolean {
+  try {
+    return recordedSteps(runDir).some((s) => s.op !== "navigate") || recordedChecks(runDir).length > 0
+  } catch {
+    return false
+  }
+}
+
+/**
+ * Replay the run's steps in Playwright: first with Playwright's default launch, as a plain test gets it; when that
+ * does not reach the run's final page, again without Playwright's automation switches, as ForgeQE's browser runs. Files go to
+ * `<run>/replay/`: flow.mjs (the reference script), one result file per setup, and a screenshot and page snapshot
+ * for each failed step.
+ */
+export async function replayRun(opts: {
+  runDir: string
+  signal?: AbortSignal
+  log?: Log
+  /** playwright-core's entry point; default: ensurePlaywright. */
+  playwright?: string
+  timeoutMs?: number
+  /** Variables the replay must not see: ForgeQE's own secrets (privateEnvNames). */
+  hide?: string[]
+}): Promise<Replay> {
+  const log = opts.log || (() => {})
+  const dir = path.join(opts.runDir, "replay")
+  const script = path.join(dir, "flow.mjs")
+  const run = readJson<{ browser?: BrowserProfile; final_url?: string }>(path.join(opts.runDir, "run.json")) || {}
+  const steps = recordedSteps(opts.runDir)
+  const replay: Replay = { dir, script, steps: steps.length, passes: [] }
+  if (!canReplay(opts.runDir)) return { ...replay, error: "the run recorded no steps to replay" }
+  let playwright = opts.playwright
+  try {
+    playwright ||= await ensurePlaywright(log)
+  } catch (e) {
+    return { ...replay, error: (e as Error).message }
+  }
+  rmSync(dir, { recursive: true, force: true })
+  mkdirSync(dir, { recursive: true })
+  const files = replayScript(steps)
+  writeFileSync(script, files.script)
+  writeFileSync(path.join(dir, "alternatives.mjs"), files.alternatives)
+  const runner = path.join(dir, "runner.mjs")
+  writeFileSync(runner, runnerSource(playwright))
+  for (const setup of ["playwright", "forgeqe"] as const) {
+    if (opts.signal?.aborted) break
+    const pass = await runPass({
+      dir,
+      runner,
+      setup,
+      browser: run.browser || {},
+      finalUrl: run.final_url || "",
+      checks: recordedChecks(opts.runDir),
+      signal: opts.signal,
+      timeoutMs: opts.timeoutMs || PASS_TIMEOUT_MS,
+      hide: opts.hide || [],
+    })
+    replay.passes.push(pass)
+    log(passLine(pass, steps.length))
+    if (pass.reached || (pass.error && !pass.steps.length)) break
+  }
+  writeFileSync(path.join(dir, "replay.json"), `${JSON.stringify(replay, null, 2)}\n`)
+  return replay
+}
+
+/** The replay an earlier, cut-off job of this run finished, if any. */
+export function savedReplay(runDir: string): Replay | null {
+  const r = readJson<Replay>(path.join(runDir, "replay", "replay.json"))
+  return r && Array.isArray(r.passes) ? r : null
+}
+
+const failedSteps = (p: ReplayPass) => p.steps.filter((s) => !s.ok && !s.skipped)
+
+/** Some steps failed, yet every step after them ran and the script still reached the run's final page. */
+function reachedPastFailures(p: ReplayPass): boolean {
+  return !p.ok && !p.error && p.reached && failedSteps(p).length > 0 && !p.steps.some((s) => s.skipped && !s.ok)
+}
+
+/** One line for the progress feed, e.g. `Replay in Playwright (default launch): 21 of 21 steps worked in 38s`. */
+export function passLine(p: ReplayPass, total: number): string {
+  const worked = p.steps.filter((s) => s.ok && !s.skipped).length
+  const where = p.setup === "playwright" ? "default launch" : "without automation switches"
+  const secs = Math.round(p.ms / 1000)
+  if (p.error && !p.steps.length) return `Replay in Playwright (${where}) did not run: ${p.error}`
+  const failed = failedSteps(p).map((s) => s.n)
+  const end = !p.reached ? ", did not reach the run's final page" : reachedPastFailures(p) ? ", still reached the run's final page" : ""
+  const checks = p.checks?.length ? `; ${p.checks.filter((c) => c.matches > 0).length} of ${p.checks.length} checks found on the last page` : ""
+  return `Replay in Playwright (${where}): ${worked} of ${total} steps worked${failed.length ? `, step${failed.length === 1 ? "" : "s"} ${failed.join(", ")} failed` : ""}${end}${checks} in ${secs}s`
+}
+
+/** The replay has a failure the coding agent should look at in the live browser. */
+export function replayFailed(r: Replay | null): boolean {
+  return Boolean(r && !r.error && r.passes.length && !r.passes[0].ok && !reachedPastFailures(r.passes[0]))
+}
+
+/** The dialogs open at that moment, from Playwright's page snapshot: a pop-up is the usual reason a click does not land. */
+function openDialogs(snapshot: string): string[] {
+  return snapshot
+    .split("\n")
+    .map((l) => l.trim().match(/^- (?:'|")?(alertdialog|dialog)(?: "((?:[^"\\]|\\.)*)")?/))
+    .flatMap((m) => (m ? [`${m[1]}${m[2] ? ` "${m[2].slice(0, 100)}"` : ""}`] : []))
+}
+
+function stepDetail(s: ReplayStep): string[] {
+  const lines = [`- Step ${s.n}, ${s.action}: ${s.error}`]
+  if (s.url) lines.push(`  Page: ${hostPath(s.url)}`)
+  if (s.alternatives?.length) lines.push(`  This element's other locators at that moment: ${s.alternatives.map((a) => `${a.locator} matched ${a.matches < 0 ? "an error" : a.matches}`).join("; ")}`)
+  if (s.events?.length) lines.push(`  While it ran: ${s.events.join(" | ")}`)
+  const files = [s.screenshot && `screenshot ${s.screenshot}`, s.snapshot && `page snapshot ${s.snapshot}`].filter(Boolean)
+  if (files.length) lines.push(`  Saved: ${files.join(", ")}`)
+  const dialogs = s.snapshot && existsSync(s.snapshot) ? openDialogs(readFileSync(s.snapshot, "utf8")) : []
+  if (dialogs.length) lines.push(`  Open on the page at that moment: ${dialogs.join(", ")}`)
+  return lines
+}
+
+function passSummary(p: ReplayPass, total: number): string {
+  if (p.error && !p.steps.length) return `it did not run: ${p.error}.`
+  const failed = failedSteps(p)
+  const notRun = p.steps.filter((s) => s.skipped && !s.ok).length
+  const parts = [
+    failed.length ? `${failed.length === 1 ? "step" : "steps"} ${failed.map((s) => s.n).join(", ")} failed` : `all ${total} steps worked`,
+    notRun ? `${notRun} after them were not run` : "",
+    p.error ? `it stopped: ${p.error}` : "",
+    p.reached ? `it reached the run's final page (${hostPath(p.finalUrl)})` : `it ended on ${p.finalUrl ? hostPath(p.finalUrl) : "no page"}, not the run's final page`,
+    `navigator.webdriver was ${p.webdriver}`,
+  ]
+  return `${parts.filter(Boolean).join("; ")} (${Math.round(p.ms / 1000)}s).`
+}
+
+/** The handoff's section on the replay: what worked in Playwright, and for a failed step, what the page showed. */
+export function replaySection(r: Replay): string[] {
+  const lines = ["## Replay in Playwright", ""]
+  if (r.error || !r.passes.length) {
+    lines.push(`ForgeQE could not replay the steps in Playwright: ${r.error || "no result"}. Check the steps on the live page yourself.`)
+    return lines
+  }
+  const [first, second] = r.passes
+  const skipped = first.steps.filter((s) => s.skipped && s.ok)
+  lines.push(
+    `ForgeQE ran the steps above as a Playwright script (playwright-core ${PLAYWRIGHT_VERSION}, the run's browser channel, headless or headed, user agent, language, time zone, and viewport; ${STEP_TIMEOUT_MS / 1000}s per step). Script: ${r.script}`,
+    "",
+    `- ${SETUP_LABEL.playwright}: ${passSummary(first, r.steps)}`,
+  )
+  if (second) lines.push(`- ${SETUP_LABEL.forgeqe}: ${passSummary(second, r.steps)}`)
+  if (skipped.length) lines.push(`- Not replayed (no Playwright equivalent recorded): ${skipped.map((s) => `step ${s.n} (${s.action.replace(/^not replayed: /, "")})`).join(", ")}.`)
+  const pages = [second, first].find((p) => p?.reached && p.pages?.length)?.pages || first.pages || []
+  if (pages.length) {
+    lines.push(
+      `- Page snapshots (roles, accessible names, and text as Playwright sees them, each page as the replay left it; search them for the element you need): ${pages.map((p) => `${p.url}: ${p.snapshot}`).join("; ")}`,
+    )
+  }
+  lines.push("")
+  const port =
+    "Port its steps into the repository's framework and page objects with the same locators and waits, then add the goal's assertions (the handoff's checks, when it lists them). Steps that dealt with a pop-up or banner on this run (see the browser agent's notes) still belong in a handler, as the rules say. Steps that run the browser agent's JavaScript were how it looked around or read values, not steps to copy: where the JavaScript clicks or types, do that with a Playwright action on the same element; where it reads values, assert them with locators."
+  if (first.ok) {
+    lines.push(`The script works as it is. ${port}`)
+    return lines
+  }
+  const failed = failedSteps(first)
+  for (const s of failed) lines.push(...stepDetail(s))
+  if (reachedPastFailures(first)) {
+    const which = `Step${failed.length === 1 ? "" : "s"} ${failed.map((s) => s.n).join(", ")}`
+    lines.push(
+      "",
+      `${which} failed, but every step after ${failed.length === 1 ? "it" : "them"} worked and the script still reached the run's final page, so the rest of the script works as it is. Decide from the step and the browser agent's note whether the goal needs it: if it does, find its element in the page snapshot; if not (it changed nothing on the run), leave it out. ${port}`,
+    )
+    return lines
+  }
+  if (second?.ok) {
+    lines.push(
+      "",
+      "Without Playwright's automation switches every step worked, so the difference is the browser setup, not the steps. Put the same launch settings in the repository's browser configuration (Playwright: ignoreDefaultArgs [\"--enable-automation\"] and args [\"--disable-blink-features=AutomationControlled\"]; .NET: BrowserTypeLaunchOptions.IgnoreDefaultArgs and Args), for the whole suite.",
+    )
+  } else if (second) {
+    const again = failedSteps(second).map((s) => s.n)
+    lines.push(
+      "",
+      again.length
+        ? `Without the automation switches step${again.length === 1 ? "" : "s"} ${again.join(", ")} failed too, so the launch switches are not the cause.`
+        : "Without the automation switches the replay got further; compare the two results.",
+    )
+  }
+  lines.push(
+    "",
+    "The steps before the first failure work as written. Look at the failing steps first (the saved screenshot and page snapshot, and your live_browser_* tools on that page), change what each needs, and then write the test.",
+  )
+  return lines
+}