@capagents/forgeqe 0.0.0-stage → 1.0.1

package/src/code/loop.ts

@@ -0,0 +1,603 @@
+import { existsSync, mkdirSync, readFileSync, realpathSync, rmSync, statSync, writeFileSync } from "node:fs"
+import path from "node:path"
+
+import type { CodeEvent, CodeResult, CodeReview, Usage } from "../types.ts"
+import { privateEnvNames } from "../childEnv.ts"
+import { addCost, addUsage, costOf } from "../usage.ts"
+import type { AppConfig } from "../config.ts"
+import { buildHandoff, type KnownDefect } from "./handoff.ts"
+import type { TestCase } from "../cases/types.ts"
+import { buildProfile, profileSection } from "./profile.ts"
+import { LIVE_BROWSER_SERVER, liveBrowser } from "./livebrowser.ts"
+import { addWrittenTest, notesSection, readNotes, saveNotes } from "./notes.ts"
+import { canReplay, replayRun, savedReplay, type Replay } from "./replay.ts"
+import { openSession, READ_AGENT, TEST_AGENT, type OpencodeSession, type PromptOutcome } from "./runtime.ts"
+import { failureContext } from "./slices.ts"
+import { moveToAttention } from "./attention.ts"
+import { recordMapping } from "./traceability.ts"
+import {
+  blockedChallengePrompt,
+  carryOnPrompt,
+  firstPrompt,
+  browserMismatch,
+  fixPrompt,
+  LIVE_ON_NOTE,
+  missingCommandPrompt,
+  nextAttemptPrompt,
+  noBrowserPrompt,
+  preparePrompt,
+  repeatedFailurePrompt,
+  restoredNote,
+  RESUMED_NOTE,
+  reviewFixPrompt,
+  stallPrompt,
+  standardsFixPrompt,
+  type AgentReport,
+  type ExistingTest,
+} from "./prompt.ts"
+import { openReviewer, type Reviewer } from "./review.ts"
+import { runTestCommand } from "./testrun.ts"
+import { changedFiles, jobChanges, ownPaths, preexisting, protectPatterns, restoreRemoved, restoreSnapshot, snapshot, snapshotFiles } from "./worktree.ts"
+import { loadSkills } from "../skills/registry.ts"
+import { domainNotes } from "../domain.ts"
+
+function realpathOr(dir: string): string {
+  try {
+    return realpathSync(dir)
+  } catch {
+    return path.resolve(dir)
+  }
+}
+
+/** Reviewer rejections of a passing test before the job stops. */
+const MAX_REJECTIONS = 2
+
+const FATAL_ERRORS = ["ProviderAuthError", "APIError", "ContextOverflowError", "ContentFilterError"]
+/** Turn errors after which the same session is worth reopening on a rerun (the model was unreachable or refused the key). */
+const RESUMABLE_ERRORS = ["ProviderAuthError", "APIError"]
+
+export const SESSION_FILE = "scribe-session.json"
+type SavedSession = { sessionId: string; repo: string; stage: "reading" | "writing"; liveOn?: boolean; at?: string }
+
+/** Scribe's session on this run and repo, left by a job that stopped before it finished. */
+export function savedSession(runDir: string, repo: string): SavedSession | null {
+  try {
+    const s = JSON.parse(readFileSync(path.join(runDir, SESSION_FILE), "utf8")) as SavedSession
+    return s.sessionId && s.repo === repo ? s : null
+  } catch {
+    return null
+  }
+}
+
+/** A test run's output without its numbers (timings, line numbers, counts), to tell the same failure again. */
+/** One line of the attempt log a fresh session starts from: what ran, how it ended, and what Scribe said it did. */
+function attemptLine(attempt: number, outcome: string, detail: string, notes?: string): string {
+  const clip = (t: string, n: number) => (t.length > n ? `${t.slice(0, n - 1)}…` : t)
+  return [`${attempt}: ${outcome}`, detail ? `: ${clip(detail.replace(/\s+/g, " ").trim(), 240)}` : "", notes ? ` (Scribe: ${clip(notes.replace(/\s+/g, " ").trim(), 240)})` : ""].join("")
+}
+
+/** The first line of a test run's output that names the error. */
+function firstError(output: string): string {
+  const lines = output.split("\n").map((l) => l.trim()).filter(Boolean)
+  return lines.find((l) => /error|exception|failed|timeout|expected|strict mode/i.test(l) && !/^at /.test(l)) || lines[0] || ""
+}
+
+export function failureKey(output: string): string {
+  return output.replace(/\d+/g, "#").replace(/\s+/g, " ").trim()
+}
+
+/**
+ * Hand a passed browser run to Scribe and loop until the test it writes passes when ForgeQE runs it.
+ * The agent's own claim of success does not count. An attempt passes only when ForgeQE's run of the reported
+ * command exits 0 with a browser launched under it, and a read-only reviewer approves the change against the goal.
+ */
+export async function runCodeAgent(opts: {
+  config: AppConfig
+  repo: string
+  runDir: string
+  signal?: AbortSignal
+  onEvent?: (e: CodeEvent) => void
+  /** Replaces openSession; tests use it to script the agent. */
+  open?: (o: Parameters<typeof openSession>[0]) => Promise<OpencodeSession>
+  /** Replaces runTestCommand. */
+  runTest?: typeof runTestCommand
+  /** Replaces the Scribe reviewer. */
+  reviewer?: Reviewer
+  /** Replaces replayRun. */
+  replay?: typeof replayRun
+  /** The test case the run came from: the test is named and tagged for it, and mapped to it once it passes. */
+  testCase?: TestCase
+  /** The run found a functional defect: the test asserts the correct behaviour, marked as an expected failure. */
+  knownDefect?: KnownDefect
+  /** The test the case already has, which failed when ForgeQE ran it: the job updates it. */
+  existingTest?: ExistingTest
+}): Promise<CodeResult> {
+  const given = path.resolve(opts.repo)
+  if (!existsSync(given) || !statSync(given).isDirectory()) throw new Error(`Repo not found: ${given}`)
+  const repo = realpathSync(given)
+  const { config } = opts
+  const emit = opts.onEvent || (() => {})
+  let handoff = buildHandoff(opts.runDir, config.memory.dir, { knownDefect: opts.knownDefect })
+  if (!handoff.success && !opts.knownDefect) throw new Error("The coding agent only takes runs that passed. This run failed.")
+  const plural = (n: number, w: string) => `${n} ${w}${n === 1 ? "" : "s"}`
+  const shown = path.relative(process.cwd(), handoff.file)
+  const handed = [plural(handoff.steps, "step"), handoff.checks ? plural(handoff.checks, "check") : "", handoff.pages ? `${plural(handoff.pages, "page")} from page memory` : ""]
+  emit({ type: "code", kind: "info", text: `Handoff written: ${shown && !shown.startsWith("..") ? shown : handoff.file} (${handed.filter(Boolean).join(", ")})` })
+  const maxAttempts = Math.max(1, config.code.maxAttempts)
+
+  const result: CodeResult = {
+    status: "failed",
+    repo,
+    attempts: 0,
+    maxAttempts,
+    reused: [],
+    changed: [],
+    notes: "",
+    output: "",
+    usage: { input: 0, output: 0 },
+    model: "",
+    handoff: handoff.file,
+  }
+
+  const readDirs = [...new Set([given, opts.runDir, config.memory.dir].flatMap((d) => [path.resolve(d), realpathOr(d)]))]
+  const own = ownPaths(repo, [opts.runDir, config.memory.dir])
+  // How the run's browser was set up, to tell Scribe when its test's browser differs (headless, another browser).
+  const runBrowser = (() => {
+    try {
+      return (JSON.parse(readFileSync(path.join(opts.runDir, "run.json"), "utf8")) as { browser?: { headless?: boolean; channel?: string } }).browser
+    } catch {
+      return undefined
+    }
+  })()
+  const before = snapshot(repo, own)
+  const outside = [opts.runDir, config.memory.dir].map((d) => realpathOr(d)).filter((d) => path.relative(repo, d).startsWith(".."))
+  let session: OpencodeSession | null = null
+  const reviewer = opts.reviewer || openReviewer({ config, repo, readDirs, signal: opts.signal, onEvent: emit, open: opts.open })
+  const runTest = opts.runTest || runTestCommand
+  let challenged = false
+  let stalls = 0
+  let lastFailure = ""
+  let rejections = 0
+  /** The passing, goal-approved version, kept while the agent fixes coding standards in one more round. */
+  let tidy: { files: Map<string, Buffer | null>; report: AgentReport; review: CodeReview; output: string; browsers: string[] } | null = null
+  let tidying = false
+  let sameFailures = 0
+  const live = config.code.liveBrowser ? liveBrowser(opts.runDir) : null
+  let liveOn = false
+  const replaying = Boolean(config.code.replay) && canReplay(opts.runDir)
+  const tools = () => (live ? { tools: { [`${LIVE_BROWSER_SERVER}_*`]: liveOn } } : {})
+  const earlier = savedSession(opts.runDir, repo)
+  const notesDir = config.memory.enabled ? config.memory.dir : ""
+  const known = notesDir ? readNotes(notesDir, repo) : null
+  let stage: SavedSession["stage"] = "reading"
+  let keepSession = true
+  let runtimeFailed = false
+  const saveSession = () => {
+    if (!session) return
+    const saved: SavedSession = { sessionId: session.sessionId, repo, stage, liveOn, at: new Date().toISOString() }
+    try {
+      mkdirSync(opts.runDir, { recursive: true })
+      writeFileSync(path.join(opts.runDir, SESSION_FILE), `${JSON.stringify(saved, null, 2)}\n`)
+    } catch {
+      // Only a rerun after a crash reads it.
+    }
+  }
+  let replay: Replay | null = null
+  /** The replay's page snapshots, from the pass that reached the run's last page first, and its failing steps'. */
+  const replaySnapshots = (): string[] => {
+    const passes = [...(replay?.passes || [])].sort((a, b) => Number(b.reached) - Number(a.reached))
+    return passes.flatMap((p) => [...(p.pages || []).map((pg) => pg.snapshot), ...p.steps.flatMap((st) => (st.snapshot ? [st.snapshot] : []))])
+  }
+  const switchLiveOn = (): string => {
+    if (!live || liveOn) return ""
+    liveOn = true
+    saveSession()
+    emit({ type: "code", kind: "info", text: "Scribe's live browser is on to look at the failing step" })
+    return `\n\n${LIVE_ON_NOTE}`
+  }
+  const keepHandoff = () => {
+    if (existsSync(handoff.file)) return
+    mkdirSync(path.dirname(handoff.file), { recursive: true })
+    writeFileSync(handoff.file, handoff.markdown)
+  }
+  // code.max_tokens holds during an attempt too: the turn is stopped as soon as the job's total passes it, instead of
+  // after the attempt (one long attempt can use millions of tokens on its own).
+  let overAt: number | null = null
+  const watchBudget = (e: CodeEvent) => {
+    emit(e)
+    if (e.kind !== "usage" || !config.code.maxTokens || overAt !== null) return
+    const used = addUsage(addUsage(spent, e.usage), reviewer.usage)
+    if (used.input + used.output < config.code.maxTokens) return
+    overAt = used.input + used.output
+    void session?.abort()
+  }
+  const sessionOptions = (resume?: string) => ({
+      repo,
+      profile: config.llm,
+      domain: domainNotes(config),
+      model: config.code.model,
+      bin: config.code.opencode,
+      readDirs,
+      hide: privateEnvNames(config),
+      signal: opts.signal,
+      onEvent: watchBudget,
+      protect: protectPatterns(repo, own, outside),
+      stallAfter: config.code.stallAfter ?? 5,
+      agent: { ...TEST_AGENT, steps: config.code.maxSteps ?? 50 },
+      agents: [READ_AGENT],
+      skills: loadSkills(config.skills, repo).skills,
+      ...(live ? { mcp: { [LIVE_BROWSER_SERVER]: live.config } } : {}),
+      ...(resume ? { resume } : {}),
+    })
+  // Each attempt after the first runs in a new session from a brief of the job so far (nextAttemptPrompt), so the
+  // model is not sent every earlier attempt's tool calls again. Usage of closed sessions is kept here.
+  let spent: Usage = { input: 0, output: 0 }
+  const attemptLog: string[] = []
+  let nextTask: string | null = null
+  let notesText: string | undefined = known ? notesSection(known) : undefined
+  // The repository's test setup, read by ForgeQE without a model; Scribe starts from it instead of a reading turn.
+  const repoProfile = buildProfile(repo)
+  const profileText = repoProfile ? profileSection(repoProfile) : undefined
+  try {
+    session = await (opts.open || openSession)(sessionOptions(earlier?.sessionId))
+    result.model = session.model
+    result.sessionId = session.sessionId
+    const resumed = Boolean(earlier && session.resumed)
+    if (resumed) result.resumed = true
+    if (earlier) {
+      emit({
+        type: "code",
+        kind: "info",
+        text: resumed ? "Scribe reopens its session from the earlier job on this run, which stopped before it finished" : "Scribe's session from the earlier job on this run is gone; it starts a new one",
+      })
+    }
+    saveSession()
+    emit({ type: "code", kind: "info", text: `Scribe, the coding agent (${session.model}), reads the handoff and writes a test in ${repo}` })
+
+    let prepared = true
+    const reads = !resumed && !known && !repoProfile
+    if (repoProfile && !resumed) {
+      emit({
+        type: "code",
+        kind: "info",
+        text: `ForgeQE read the repository's test setup itself (${repoProfile.pageObjects.length} page objects, ${repoProfile.baseClasses.length} base classes, ${repoProfile.settings.length} settings files${repoProfile.testIdAttribute ? `, test ids in ${repoProfile.testIdAttribute.value}` : ""}); Scribe starts from it`,
+      })
+    }
+    if (known && !resumed) {
+      const n = known.tests.length
+      emit({ type: "code", kind: "info", text: `Scribe starts from its notes on this repository (${known.at.slice(0, 10)}${n ? `, ${plural(n, "test")} written before` : ""}) instead of reading it again` })
+    }
+    const saved = replaying && resumed ? savedReplay(opts.runDir) : null
+    if (replaying) {
+      const replayText = reads ? "ForgeQE replays the run's steps in Playwright while Scribe reads the repository" : "ForgeQE replays the run's steps in Playwright"
+      emit({ type: "code", kind: "info", text: saved ? "ForgeQE uses the Playwright replay the earlier job finished" : replayText })
+    }
+    const [r, prep]: [Replay | null, PromptOutcome | null] = await Promise.all([
+      !replaying ? null : saved || (opts.replay || replayRun)({ runDir: opts.runDir, signal: opts.signal, hide: privateEnvNames(config), log: (text) => emit({ type: "code", kind: "info", text }) }),
+      reads ? session.prompt(preparePrompt(handoff.goal, replaying), { structured: false, agent: READ_AGENT.name }) : null,
+    ])
+    if (prep && !prep.error && notesDir) saveNotes(notesDir, repo, prep.text)
+    if (prep && !prep.error && prep.text.trim()) notesText = `Your notes on this repository, from reading it at the start of this job:\n${prep.text.trim()}`
+    if (r) {
+      replay = r
+      result.replay = {
+        dir: r.dir,
+        ...(r.error ? { error: r.error } : {}),
+        passes: r.passes.map((p) => ({ setup: p.setup, failed: p.steps.filter((s) => !s.ok && !s.skipped).map((s) => s.n), reached: p.reached })),
+      }
+      handoff = buildHandoff(opts.runDir, config.memory.dir, { replay, knownDefect: opts.knownDefect })
+      // The replay's failure (its error, screenshot, and page snapshot) is in the handoff; the live browser waits for a repeated failure.
+      liveOn = Boolean(live) && Boolean(resumed && earlier?.liveOn)
+    } else if (resumed && earlier?.liveOn) {
+      liveOn = Boolean(live)
+    }
+    if (opts.signal?.aborted) {
+      result.status = "stopped"
+      prepared = false
+    } else if (prep?.error && FATAL_ERRORS.some((name) => prep.error!.startsWith(name))) {
+      result.notes = prep.error
+      runtimeFailed = RESUMABLE_ERRORS.some((name) => prep.error!.startsWith(name))
+      emit({ type: "code", kind: "info", text: `Scribe stopped: ${prep.error}` })
+      prepared = false
+    }
+    if (live && prepared) {
+      emit({
+        type: "code",
+        kind: "info",
+        text: liveOn ? "Scribe's live browser is on, as it was when the earlier job stopped" : "Scribe gets a live browser (Playwright, set up like ForgeQE's) if a test failure repeats",
+      })
+    }
+    let prompt =
+      resumed && earlier?.stage === "writing"
+        ? carryOnPrompt(liveOn)
+        : `${resumed ? `${RESUMED_NOTE}\n\n` : ""}${firstPrompt(handoff, { ...(live ? { liveBrowserDir: live.dir, liveOn } : {}), ...(known ? { notes: notesSection(known) } : {}), ...(profileText ? { profile: profileText } : {}), ...(opts.existingTest ? { existing: opts.existingTest } : {}) })}`
+    if (prepared) {
+      stage = "writing"
+      saveSession()
+    }
+    let report: AgentReport = {}
+    let note = ""
+    const overBudget = (): number | null => {
+      if (!config.code.maxTokens || !session) return null
+      const used = addUsage(addUsage(spent, session.usage), reviewer.usage)
+      return used.input + used.output >= config.code.maxTokens ? used.input + used.output : null
+    }
+    /** The tidy-up round broke the test: put back the version that passed, and pass with the standards issues noted. */
+    const finishWithKept = (attempt: number, why: string) => {
+      restoreSnapshot(repo, before, tidy!.files)
+      report = { ...tidy!.report }
+      result.testFile = report.test_file
+      result.command = report.command
+      result.changed = report.changed || []
+      result.output = tidy!.output
+      result.browsers = tidy!.browsers
+      result.status = "passed"
+      const left = tidy!.review.issues.filter((i) => i.kind === "standards")
+      result.review = { ...tidy!.review, approved: true, standardsLeft: true }
+      result.notes = `The test passes and tests the goal. ${why[0].toUpperCase()}${why.slice(1)}, so ForgeQE put back the version that passed. Coding standards still to fix: ${left.map((i) => `${i.file ? `${i.file}: ` : ""}${i.problem}`).join(" ")}`
+      if (notesDir && report.test_file && report.command) addWrittenTest(notesDir, repo, { goal: handoff.goal, test_file: report.test_file, command: report.command, changed: result.changed })
+      emit({ type: "code", kind: "test", ok: true, attempt, text: `${why}; kept the version that passed (${report.test_file}), with the coding-standard issues in the notes` })
+    }
+    // The tidy-up round for coding standards does not count against max_attempts.
+    let allowed = maxAttempts
+    for (let attempt = 1; prepared && attempt <= allowed; attempt++) {
+      const over = overBudget()
+      if (over !== null) {
+        result.notes = `Stopped: the job used ${over.toLocaleString("en")} tokens, over code.max_tokens (${config.code.maxTokens!.toLocaleString("en")}). ${result.notes || ""}`.trim()
+        emit({ type: "code", kind: "test", ok: false, attempt, text: `stopped: over the token budget (code.max_tokens ${config.code.maxTokens!.toLocaleString("en")})` })
+        break
+      }
+      result.attempts = attempt
+      if (nextTask !== null) {
+        spent = addUsage(spent, session!.usage)
+        await session!.close()
+        session = await (opts.open || openSession)(sessionOptions())
+        result.sessionId = session.sessionId
+        saveSession()
+        emit({ type: "code", kind: "info", text: `Scribe starts attempt ${attempt} in a fresh session, from a summary of the job so far` })
+        prompt = nextAttemptPrompt({
+          handoff,
+          notes: [profileText, notesText].filter(Boolean).join("\n\n") || undefined,
+          testFile: report.test_file,
+          command: report.command,
+          changed: changedFiles(repo, jobChanges(before) || report.changed || []),
+          attempts: attemptLog,
+          task: nextTask,
+          ...(live ? { liveBrowserDir: live.dir, liveOn } : {}),
+        })
+        nextTask = null
+      }
+      const reply = await session!.prompt(note + prompt, tools())
+      if (overAt !== null && !opts.signal?.aborted) {
+        restoreRemoved(before)
+        keepHandoff()
+        result.notes = `Stopped: the job used ${(overAt as number).toLocaleString("en")} tokens, over code.max_tokens (${config.code.maxTokens!.toLocaleString("en")}), during attempt ${attempt}. ${report.notes || ""}`.trim()
+        emit({ type: "code", kind: "test", ok: false, attempt, text: `stopped: over the token budget (code.max_tokens ${config.code.maxTokens!.toLocaleString("en")}) during attempt ${attempt}` })
+        break
+      }
+      const restored = restoreRemoved(before)
+      keepHandoff()
+      if (restored.length) emit({ type: "code", kind: "info", text: `Put back ${restored.join(", ")}: the coding agent removed or reverted files that existed before the job` })
+      note = restored.length ? `${restoredNote(restored)}\n\n` : ""
+      if (opts.signal?.aborted) {
+        result.status = "stopped"
+        break
+      }
+      if (reply.stalled) {
+        const { command, count } = reply.stalled
+        if (stalls++ === 0) {
+          emit({ type: "code", kind: "test", ok: false, attempt, text: `${command} failed ${count} times; the agent is asked to change approach` })
+          const on = switchLiveOn()
+          prompt = stallPrompt(reply.stalled, Boolean(live)) + on
+          session.resetStall?.()
+          continue
+        }
+        result.notes = `Stopped: the coding agent ran ${command} ${count} times and it failed each time, again after being asked to change approach. Last output:\n${reply.stalled.output}`
+        emit({ type: "code", kind: "test", ok: false, attempt, text: `stopped: ${command} kept failing (${count} times) after the agent was asked to change approach` })
+        break
+      }
+      if (reply.report) report = { ...report, ...reply.report }
+      result.notes = report.notes || reply.text.slice(0, 500)
+      if (reply.error && FATAL_ERRORS.some((name) => reply.error!.startsWith(name))) {
+        result.notes = reply.error
+        runtimeFailed = RESUMABLE_ERRORS.some((name) => reply.error!.startsWith(name))
+        emit({ type: "code", kind: "info", text: `Scribe stopped: ${reply.error}` })
+        break
+      }
+      if (report.status === "blocked" && !challenged) {
+        challenged = true
+        report = { ...report, status: undefined }
+        prompt = blockedChallengePrompt(handoff) + switchLiveOn()
+        emit({ type: "code", kind: "test", ok: false, attempt, text: `reported blocked; ForgeQE's browser reached the app, so the agent compares the two browsers first` })
+        continue
+      }
+      if (report.status === "blocked") {
+        result.status = "blocked"
+        result.notes = report.blocked_reason || report.notes || result.notes
+        result.testFile = report.test_file
+        result.changed = report.changed || []
+        emit({ type: "code", kind: "test", ok: false, attempt, text: `blocked: ${result.notes || "the application blocked the test"}` })
+        break
+      }
+      if (!report.command || !report.test_file) {
+        prompt = missingCommandPrompt()
+        emit({ type: "code", kind: "test", ok: false, attempt, text: "no test command reported yet" })
+        continue
+      }
+      result.testFile = report.test_file
+      result.command = report.command
+      result.reused = report.reused || []
+      result.changed = report.changed || []
+
+      emit({ type: "code", kind: "info", text: `ForgeQE runs the test: ${report.command}` })
+      const run = await runTest({
+        command: report.command,
+        cwd: repo,
+        timeoutSeconds: config.code.testTimeoutSeconds,
+        signal: opts.signal,
+        save: path.join(opts.runDir, "test-runs", `attempt-${attempt}`),
+        hide: privateEnvNames(config),
+      })
+      // The failing element on the test's page and on the replay's: read from saved snapshots, not a live browser.
+      const files = { log: run.log, pageSnapshot: run.pageSnapshot, context: run.ok ? [] : failureContext(run.output, run.pageSnapshot, replaySnapshots()) }
+      session.resetStall?.()
+      result.output = run.output
+      result.browsers = run.browsers
+      if (opts.signal?.aborted) {
+        result.status = "stopped"
+        break
+      }
+      const secs = Math.round(run.ms / 1000)
+      if (run.ok) lastFailure = ""
+      if (tidying && run.ok && !run.browsers.length && !report.remote_browser) {
+        finishWithKept(attempt, "after the coding-standard fix the test opened no browser")
+        break
+      }
+      if (run.ok && !run.browsers.length && !report.remote_browser) {
+        emit({ type: "code", kind: "test", ok: false, attempt, text: `${report.test_file} exited 0 in ${secs}s but opened no browser (attempt ${attempt}/${maxAttempts})` })
+        attemptLog.push(attemptLine(attempt, "exited 0 but opened no browser", "", report.notes))
+        nextTask = noBrowserPrompt(report.command)
+        continue
+      }
+      if (run.ok) {
+        const where = run.browsers.length ? run.browsers.join(", ") : `remote browser (${report.remote_browser})`
+        emit({ type: "code", kind: "info", text: `${report.test_file} passed in ${secs}s in ${where}; a reviewer checks it against the goal` })
+        const changes = jobChanges(before) ?? undefined
+        // After a rejection or a tidy-up, the reviewer checks its own earlier issues instead of starting over.
+        const verdict = await reviewer.review({ handoff, report, output: run.output, browsers: run.browsers, seconds: secs, changes, before: preexisting(before), ...(result.review ? { earlier: result.review } : {}) })
+        if (opts.signal?.aborted) {
+          result.status = "stopped"
+          break
+        }
+        if (!verdict.review) {
+          result.notes = `The test passed, but the review did not finish: ${verdict.error}`
+          emit({ type: "code", kind: "test", ok: false, attempt, text: result.notes })
+          break
+        }
+        result.review = verdict.review
+        const correctness = verdict.review.issues.filter((i) => i.kind !== "standards")
+        const standards = verdict.review.issues.filter((i) => i.kind === "standards")
+        // Coding standards alone never fail a test that works and tests the goal.
+        if (tidying && correctness.length) {
+          // The tidy-up changed what the test does: back to the version that passed.
+          finishWithKept(attempt, `the coding-standard fix changed what the test checks (${verdict.review.summary})`)
+          break
+        }
+        if (verdict.review.approved || (!correctness.length && standards.length)) {
+          if (standards.length && !tidying && config.code.tidy !== false) {
+            tidy = { files: snapshotFiles(repo, jobChanges(before) || result.changed), report: { ...report }, review: verdict.review, output: run.output, browsers: run.browsers }
+            tidying = true
+            allowed = Math.max(allowed, attempt + 1)
+            emit({ type: "code", kind: "test", ok: true, attempt, text: `${report.test_file} passed and tests the goal; the reviewer found ${standards.length} coding-standard ${standards.length === 1 ? "issue" : "issues"}, so the agent fixes ${standards.length === 1 ? "it" : "them"} in one more round (the passing version is kept if that breaks it)` })
+            attemptLog.push(attemptLine(attempt, "passed and tests the goal; coding-standard issues to fix", standards.map((i) => i.problem).join(" "), report.notes))
+            nextTask = standardsFixPrompt(standards)
+            continue
+          }
+          result.status = "passed"
+          if (standards.length) {
+            result.review = { ...verdict.review, approved: true, standardsLeft: true }
+            result.notes = `The test passes and tests the goal. Coding standards still to fix: ${standards.map((i) => `${i.file ? `${i.file}: ` : ""}${i.problem}`).join(" ")}`
+          } else result.review = { ...verdict.review, approved: true }
+          if (notesDir) addWrittenTest(notesDir, repo, { goal: handoff.goal, test_file: report.test_file, command: report.command, changed: result.changed })
+          emit({ type: "code", kind: "test", ok: true, attempt, text: `${report.test_file} passed and was approved (attempt ${attempt}/${allowed})${standards.length ? `; ${standards.length} coding-standard ${standards.length === 1 ? "issue" : "issues"} left in the notes` : ""}` })
+          break
+        }
+        const n = verdict.review.issues.length
+        emit({ type: "code", kind: "test", ok: false, attempt, text: `review found ${n} ${n === 1 ? "issue" : "issues"}: ${verdict.review.summary} (attempt ${attempt}/${maxAttempts})` })
+        // A second rejection usually means the goal and the page disagree (a test case the page cannot satisfy), not a
+        // fix the next attempt will find: stop and report the issues instead of spending the remaining attempts.
+        if (++rejections >= MAX_REJECTIONS) {
+          result.notes = `Stopped: the test passes, but the reviewer did not approve it ${rejections} times. Latest review: ${verdict.review.summary}`
+          emit({ type: "code", kind: "test", ok: false, attempt, text: `stopped: the reviewer did not approve the test ${rejections} times; its issues are in the result` })
+          break
+        }
+        attemptLog.push(attemptLine(attempt, "passed; the reviewer asked for changes", verdict.review.summary, report.notes))
+        nextTask = reviewFixPrompt(verdict.review.summary, verdict.review.issues)
+        continue
+      }
+      if (tidying) {
+        finishWithKept(attempt, `the coding-standard fix made the test fail (${run.exitCode === null ? "timed out" : `exit ${run.exitCode}`})`)
+        break
+      }
+      const why = run.exitCode === null ? `timed out after ${config.code.testTimeoutSeconds}s` : `exit ${run.exitCode}`
+      const mismatch = browserMismatch(runBrowser, run.browsers, report.command)
+      if (mismatch) emit({ type: "code", kind: "info", text: `The test's browser differs from the run's: ${run.browsers.join(", ")}` })
+      const key = failureKey(run.output)
+      sameFailures = key && key === lastFailure ? sameFailures + 1 : 0
+      lastFailure = key
+      if (sameFailures >= 2) {
+        result.notes = `Stopped: the test failed the same way ${sameFailures + 1} times in a row (apart from timings and line numbers), again after the agent was asked to look at the failing step first. Last output:\n${run.output.split("\n").slice(-40).join("\n")}`
+        emit({ type: "code", kind: "test", ok: false, attempt, text: `stopped: ${report.test_file} failed the same way ${sameFailures + 1} times in a row` })
+        break
+      }
+      if (sameFailures === 1) {
+        emit({ type: "code", kind: "test", ok: false, attempt, text: `${report.test_file} failed the same way as the last attempt, ${why} (attempt ${attempt}/${maxAttempts})` })
+        attemptLog.push(attemptLine(attempt, `failed, ${why}, the same way as before`, firstError(run.output), report.notes))
+        nextTask = repeatedFailurePrompt(sameFailures + 1, report.command, run.exitCode, run.output, files) + mismatch + switchLiveOn()
+        continue
+      }
+      emit({ type: "code", kind: "test", ok: false, attempt, text: `${report.test_file} failed, ${why} (attempt ${attempt}/${maxAttempts})` })
+      attemptLog.push(attemptLine(attempt, `failed, ${why}`, firstError(run.output), report.notes))
+      nextTask = fixPrompt(attempt, report.command, run.exitCode, run.output, files) + mismatch
+    }
+    keepSession = result.status === "stopped" || runtimeFailed
+    if (report.test_name) result.testName = report.test_name
+    if (opts.knownDefect) result.knownDefect = true
+    const tc = opts.testCase
+    if (result.status === "passed" && tc) {
+      recordMapping(repo, {
+        caseId: tc.id,
+        title: tc.title,
+        source: tc.source,
+        ...(tc.url ? { url: tc.url } : {}),
+        status: opts.knownDefect ? "known_defect" : "automated",
+        ...(result.testFile ? { testFile: result.testFile } : {}),
+        ...(result.testName ? { testName: result.testName } : {}),
+        ...(result.command ? { command: result.command } : {}),
+        ...(opts.knownDefect?.bug ? { bug: opts.knownDefect.bug } : {}),
+      })
+    }
+    // An unfinished job (not one that can be resumed) leaves no failing test among the tests: it goes to needs-attention.
+    if ((result.status === "failed" || result.status === "blocked") && !keepSession && config.code.needsAttention !== false) {
+      const attention = moveToAttention({
+        repo,
+        tree: before,
+        testFile: result.testFile,
+        command: result.command,
+        caseId: tc?.id,
+        title: tc?.title,
+        url: tc?.url,
+        why: result.notes || `The test did not pass after ${result.attempts} attempts.`,
+        output: result.output,
+        review: result.review,
+      })
+      if (attention) {
+        result.attention = attention
+        emit({ type: "code", kind: "info", text: `The unfinished test was moved to ${attention.dir}/ with a README on why and how to pick it up${attention.patch ? "; its edits to existing files are in changes.patch" : ""}` })
+        if (tc) recordMapping(repo, { caseId: tc.id, title: tc.title, source: tc.source, ...(tc.url ? { url: tc.url } : {}), status: "needs_attention", attention: attention.dir, ...(result.command ? { command: result.command } : {}) })
+      }
+    }
+  } finally {
+    if (session) {
+      result.usage = addUsage(spent, session.usage)
+      if (session.skillsUsed?.length) result.skills = [...session.skillsUsed]
+      await session.close()
+    }
+    await reviewer.close()
+    // With code.model the agent runtime picks the model, so there is no price for it in llms.json; the reviewer is
+    // priced at its own profile (code.review_profile). The job's cost is unknown when either part has no price.
+    const scribeCost = costOf(result.usage, config.code.model ? undefined : config.llm.price)
+    const reviewCost = reviewer.usage.input || reviewer.usage.output ? costOf(reviewer.usage, reviewer.price) : undefined
+    const reviewed = Boolean(reviewer.usage.input || reviewer.usage.output)
+    result.usage = addUsage(result.usage, reviewer.usage)
+    result.cost = scribeCost && (!reviewed || reviewCost) ? addCost(scribeCost, reviewCost) : undefined
+    if (!keepSession) rmSync(path.join(opts.runDir, SESSION_FILE), { force: true })
+    try {
+      mkdirSync(opts.runDir, { recursive: true })
+      writeFileSync(path.join(opts.runDir, "code.json"), `${JSON.stringify(result, null, 2)}\n`)
+    } catch (e) {
+      emit({ type: "code", kind: "info", text: `Could not save code.json: ${(e as Error).message}` })
+    }
+  }
+  return result
+}