@capagents/forgeqe 1.1.9 → 1.2.1

package/src/cases/quality.ts

@@ -54,7 +54,9 @@ export function checkCase(tc: TestCase): CaseFinding[] {
   }
   const withExpected = steps.filter((s) => s.expected?.trim())
   if (!withExpected.length) {
-    out.push({ code: "no_expected_results", severity: "blocker", message: "No step has an expected result, so the run cannot pass or fail on anything.", suggestion: "Give at least the steps that matter an expected result the page shows (a message, a value, a page)." })
+    // It still runs: each step is done and verified, so the case passes when every step can be done. What it does not
+    // have is a result to check, so it is worth fixing, not a reason to refuse it.
+    out.push({ code: "no_expected_results", severity: "warning", message: "No step has an expected result: the run passes when every step can be done, and checks no result the page should show.", suggestion: "Give at least the steps that matter an expected result the page shows (a message, a value, a page)." })
   }
   steps.forEach((s, i) => {
     const n = i + 1

package/src/code/coverage.ts

@@ -0,0 +1,117 @@
+import { existsSync, readFileSync } from "node:fs"
+import path from "node:path"
+
+import YAML from "yaml"
+
+import { readPages } from "../memory.ts"
+import { membersLocating, qualified, sameElement, testsReachingMembers, type CodeGraph, type Located } from "./graph/graph.ts"
+import { fromMemory, type LocatorRef } from "./graph/locators.ts"
+import type { Check, Locator, Step } from "./handoff.ts"
+
+/**
+ * What the repository's existing tests already cover of a passed run, measured rather than guessed: each step and
+ * check the run made on an element is matched, through page memory, to the page-object members that locate that
+ * element, and the code graph says which tests reach those members. From that, the smallest safe change: reuse a test
+ * that covers everything, extend one that covers part, write a new one, or ask when the evidence does not settle it.
+ */
+
+/** A step or check of the run that acted on or read an element, and the repository members that locate it. */
+export type CoverageItem = { label: string; kind: "step" | "check"; members: string[] }
+
+/** An existing test and which of the run's items (by index) it reaches. */
+export type Candidate = { test: string; file: string; line: number; covered: number[]; missing: number[]; share: number }
+
+export type Mode = "new" | "extend" | "reuse" | "update"
+
+/** `ask`: the evidence fits more than one way, so the person chooses (the options, each with what it changes). */
+export type Decision = { mode: Mode; ask: boolean; target?: Candidate; why: string }
+
+export type Coverage = { items: CoverageItem[]; candidates: Candidate[]; decision: Decision }
+
+/** A test covering at least this share of the run's items is worth extending rather than duplicating. */
+export const EXTEND_FROM = 0.4
+/** Another test this close to the best one makes the choice the person's. */
+const RIVAL = 0.75
+
+type RunJson = { final_url?: string; memory?: { dir?: string } }
+
+function readJson<T>(file: string): T | null {
+  try {
+    return JSON.parse(readFileSync(file, "utf8")) as T
+  } catch {
+    return null
+  }
+}
+
+const quote = (t?: string) => (t ? `"${t.replace(/\s+/g, " ").trim().slice(0, 60)}"` : "")
+
+/** "click "Toggle Input Box 2"", "input "Name"", "check "Interactive SVG Pie Chart"". */
+function stepLabel(s: Step & { label?: string; text?: string }): string {
+  return `${s.op} ${quote(s.label || s.text) || "(an element)"}`
+}
+
+export function measureCoverage(runDir: string, graph: CodeGraph, memoryDir?: string): Coverage {
+  const run = readJson<RunJson>(path.join(runDir, "run.json")) || {}
+  const file = path.join(runDir, "workflow.yaml")
+  const workflow = existsSync(file) ? ((YAML.parse(readFileSync(file, "utf8")) || {}) as { steps?: Step[]; checks?: Check[] }) : {}
+  const memDir = run.memory?.dir || memoryDir
+  const pages = memDir ? readPages(memDir) : []
+  const located: Located[][] = []
+  const items: CoverageItem[] = []
+  const add = (label: string, kind: CoverageItem["kind"], locators: Locator[] | undefined, page: string | undefined) => {
+    const refs = (locators || []).map((l) => fromMemory(l.type, l.value)).filter((r): r is LocatorRef => Boolean(r))
+    if (!refs.length) return
+    const same = sameElement(pages, refs, page)
+    const matches = membersLocating(graph, same.refs).filter((m) => m.how !== "param")
+    located.push(matches.map((m) => m.at))
+    items.push({ label, kind, members: [...new Set(matches.map((m) => qualified(m.at)))] })
+  }
+  for (const s of workflow.steps || []) if (["click", "input", "select"].includes(s.op || "")) add(stepLabel(s), "step", s.locators, s.page || s.page_url)
+  for (const c of workflow.checks || []) if (c?.text?.trim()) add(`check ${quote(c.expected || c.text)}`, "check", c.element?.locators, c.url || run.final_url)
+
+  // Each test, with the items it reaches.
+  const byTest = new Map<string, { at: Located; covered: Set<number> }>()
+  located.forEach((members, i) => {
+    if (!members.length) return
+    for (const t of testsReachingMembers(graph, members)) {
+      const key = `${t.file}#${qualified(t)}`
+      const entry = byTest.get(key) || { at: t, covered: new Set<number>() }
+      entry.covered.add(i)
+      byTest.set(key, entry)
+    }
+  })
+  // Only items the repository locates can be measured: a step no page-object member locates may still be done by a
+  // test in its own way, so it is unknown, not missing.
+  const measurable = items.flatMap((it, i) => (it.members.length ? [i] : []))
+  const candidates = [...byTest.values()]
+    .map(({ at, covered }) => ({ test: qualified(at), file: at.file, line: at.member.line, covered: [...covered].sort((a, b) => a - b), missing: measurable.filter((i) => !covered.has(i)), share: measurable.length ? covered.size / measurable.length : 0 }))
+    .sort((a, b) => b.covered.length - a.covered.length || a.missing.length - b.missing.length)
+  return { items, candidates, decision: decide(items, candidates) }
+}
+
+/** The smallest safe change the evidence supports, and whether it is clear enough to go ahead without asking. */
+export function decide(items: CoverageItem[], candidates: Candidate[]): Decision {
+  const best = candidates[0]
+  const n = items.filter((i) => i.members.length).length
+  if (!n || !best || best.share < EXTEND_FROM) {
+    return { mode: "new", ask: false, why: !n ? "no element this run used is located by the repository's page objects yet" : best ? `no existing test covers more than ${best.covered.length} of the run's ${n} steps and checks` : "no existing test reaches the elements this run used" }
+  }
+  const rival = candidates[1]
+  if (rival && rival.share >= EXTEND_FROM && rival.covered.length >= RIVAL * best.covered.length) {
+    return { mode: best.missing.length ? "extend" : "reuse", ask: true, target: best, why: `${candidates.filter((c) => c.share >= EXTEND_FROM).length} existing tests each cover most of this flow` }
+  }
+  if (!best.missing.length) return { mode: "reuse", ask: false, target: best, why: `${best.test} already covers all ${n} steps and checks` }
+  return { mode: "extend", ask: false, target: best, why: `${best.test} covers ${best.covered.length} of ${n}; only ${best.missing.length} ${best.missing.length === 1 ? "is" : "are"} missing` }
+}
+
+/** The evidence as card lines: each close test, what it covers, and what it lacks. */
+export function coverageLines(c: Coverage, max = 3): string[] {
+  const n = c.items.filter((i) => i.members.length).length
+  const lines = c.candidates.slice(0, max).map((t) => {
+    const missing = t.missing.map((i) => c.items[i]!.label)
+    return `${t.share >= EXTEND_FROM ? "◐" : "·"} ${t.test} (${t.file}:${t.line}) covers ${t.covered.length} of ${n}${missing.length ? `; missing: ${missing.slice(0, 4).join(", ")}${missing.length > 4 ? ` +${missing.length - 4} more` : ""}` : ""}`
+  })
+  const unlocated = c.items.filter((i) => !i.members.length)
+  if (unlocated.length) lines.push(`· not measured: ${unlocated.length} of ${c.items.length} steps ${unlocated.length === 1 ? "has" : "have"} no page-object member (${[...new Set(unlocated.map((i) => i.label))].slice(0, 3).join(", ")}); a test may do ${unlocated.length === 1 ? "it" : "them"} its own way`)
+  return lines
+}

package/src/code/exchange.ts

@@ -15,6 +15,27 @@ const clip = (t: string, n: number) => {
   return s.length <= n ? s : `${s.slice(0, n - 1)}…`
 }
 
+/**
+ * The text without the domain statement the prompts ask every agent to open with ("Assumed industry: …; no
+ * regulated-industry obligations applied."): right for the record, but in the exchange it hides what was done.
+ */
+export function withoutDomainNote(text: string): string {
+  const note = /^(Assumed (industry|domain)\b|No regulated-industry|Industry assumed\b)/i
+  const kept = text
+    .split(/(?<=[.!?])\s+/)
+    .map((sentence) => {
+      const t = sentence.trim()
+      if (!note.test(t)) return t
+      // The note, then what the agent actually says after it in the same sentence: keep that part.
+      const rest = t.split(";").slice(1).map((p) => p.trim()).filter((p) => p && !/^no regulated-industry/i.test(p)).join("; ")
+      return rest ? rest[0]!.toUpperCase() + rest.slice(1) : ""
+    })
+    .filter(Boolean)
+  // An "Approved." that only led into the domain note says nothing the title does not.
+  const out = kept.join(" ").replace(/^Approved\.\s*$/i, "").trim()
+  return out || text.trim()
+}
+
 /** "✗ Does not test the goal · tests/Practice.cs: the pie chart is never checked", then the fix under it. */
 export function reviewCard(e: Review): Card {
   const correctness = e.issues.filter((i) => i.kind !== "standards").length
@@ -25,7 +46,7 @@ export function reviewCard(e: Review): Card {
     title: e.approved ? (standards ? "Reviewer approved the test, with coding-standard notes" : "Reviewer approved the test") : "Reviewer asked for changes",
     stats: [`attempt ${e.attempt} of ${e.max}`, count].filter(Boolean).join(" · "),
     body: [
-      ...(e.summary ? [clip(e.summary, 400)] : []),
+      ...(e.summary ? [clip(withoutDomainNote(e.summary), 400)] : []),
       ...(e.issues.length ? [""] : []),
       ...e.issues.flatMap((i) => [
         // ✗ marks what makes the test wrong, ~ how it is written: the colours follow the mark.
@@ -49,5 +70,5 @@ export function attemptCard(e: Attempt): Card {
 /** "Scribe (attempt 2): Moved the values into PracticePage · changed tests/Practice.cs, pages/PracticePage.cs". */
 export function answerText(e: Answer): string {
   const files = e.changed.length ? `changed ${e.changed.slice(0, 4).join(", ")}${e.changed.length > 4 ? ` +${e.changed.length - 4} more` : ""}` : ""
-  return `Scribe (attempt ${e.attempt}): ${[e.notes ? clip(e.notes, 300) : "", files].filter(Boolean).join(" · ")}`
+  return `Scribe (attempt ${e.attempt}): ${[e.notes ? clip(withoutDomainNote(e.notes), 300) : "", files].filter(Boolean).join(" · ")}`
 }

package/src/code/graph/graph.ts

@@ -90,7 +90,7 @@ export function allMembers(g: CodeGraph): Located[] {
   return g.files.flatMap((f) => [...f.classes.flatMap((cls) => cls.members.map((member) => ({ file: f.file, cls, member }))), ...f.functions.map((member) => ({ file: f.file, member }))])
 }
 
-const qualified = (l: Located) => (l.cls ? `${l.cls.name}.${l.member.name}` : l.member.name)
+export const qualified = (l: Located) => (l.cls ? `${l.cls.name}.${l.member.name}` : l.member.name)
 const where = (file: string, line: number) => `${file}:${line}`
 
 /** Words of an identifier or phrase: SignInAsync → sign, in, async; "Sign in" → sign, in. */
@@ -397,14 +397,41 @@ export function describeReferences(target: string, r: ReturnType<typeof referenc
  */
 export function testsReaching(g: CodeGraph, changed: string[]): { test: Located; why: string }[] {
   const files = new Set(changed.map((f) => f.replace(/^\.\//, "")))
-  const classes = new Set(g.files.flatMap((f) => f.classes.map((c) => c.name)))
-  const byFile = new Map(g.files.map((f) => [f.file, f]))
   const all = allMembers(g)
   // Reached members, as Class.member, with why.
   const reached = new Map<string, string>()
   for (const l of all) if (files.has(l.file)) reached.set(qualified(l), `in ${l.file}`)
   const out = new Map<Located, string>()
   for (const l of all) if (files.has(l.file) && l.member.kind === "test") out.set(l, `is in ${l.file}`)
+  return reachTests(g, reached, out)
+}
+
+/**
+ * The tests that reach any of these members (page-object fields or methods): directly, or through the page methods
+ * that use a field and the calls that lead to them. What an existing test already exercises of a run's elements.
+ */
+export function testsReachingMembers(g: CodeGraph, members: Located[]): Located[] {
+  const reached = new Map<string, string>()
+  for (const m of members) {
+    reached.set(qualified(m), "located")
+    // A field is reached through the methods that read it (LoginButton → SignInAsync), which is what a test calls.
+    for (const user of usersOf(g, m)) if (!reached.has(user)) reached.set(user, `uses ${qualified(m)}`)
+  }
+  const out = new Map<Located, string>()
+  const all = allMembers(g)
+  // A test that reads the member as a property reaches it directly: in its own class, or through a variable of the
+  // member's page-object class (Expect(practicePage.AjaxStatus) with practicePage a PracticePage). Reads are not calls.
+  const byFile = new Map(g.files.map((f) => [f.file, f]))
+  const holds = (l: Located, cls: string) => l.cls?.name === cls || Object.values(byFile.get(l.file)?.bindings || {}).some((b) => lineage(g, b).includes(cls))
+  for (const l of all) if (l.member.kind === "test" && members.some((m) => m.cls && l.member.idents.includes(m.member.name) && holds(l, m.cls.name))) out.set(l, "reads it")
+  return reachTests(g, reached, out).map((r) => r.test)
+}
+
+/** Grows `reached` along calls, up to five levels, and adds each test that calls a reached member to `out`. */
+function reachTests(g: CodeGraph, reached: Map<string, string>, out: Map<Located, string>): { test: Located; why: string }[] {
+  const classes = new Set(g.files.flatMap((f) => f.classes.map((c) => c.name)))
+  const byFile = new Map(g.files.map((f) => [f.file, f]))
+  const all = allMembers(g)
   for (let round = 0; round < 5; round++) {
     let grew = false
     for (const l of all) {

package/src/code/handoff.ts

@@ -43,6 +43,16 @@ export type Step = {
   scripted?: boolean
   /** click: the control sat in an overlay (cookie banner, sign-in prompt, dialog) that this click dismissed. */
   overlay?: { kind?: string; name?: string; id?: string }
+  /** api: an HTTP call made in the browser's session, and what the server answered when the run made it. */
+  method?: string
+  body?: string
+  content_type?: string
+  /** api: where the CSRF token came from: the "cookie", or the page's "form". */
+  csrf?: string
+  status?: number
+  final_url?: string
+  redirected?: boolean
+  snippet?: string
 }
 
 /** An element the engine found on the live page, with its locators (verified ones first). */
@@ -174,6 +184,10 @@ function stepLines(step: Step, n: number): string[] {
       break
     case "press":
       return [`${n}. press ${step.keys}`]
+    case "api":
+      return [
+        `${n}. API ${step.method || "GET"} ${step.url}${step.body ? ` with ${step.content_type?.includes("json") ? "JSON" : "form"} body ${quote(step.body.slice(0, 300))}` : ""}${step.csrf ? `, CSRF token from the ${step.csrf === "form" ? "page's form" : "csrftoken cookie"}` : ""}: the server answered ${step.status ?? "?"}${step.redirected && step.final_url ? `, ending at ${step.final_url}` : ""}${step.snippet ? `; response starts: ${quote(step.snippet.slice(0, 160))}` : ""}`,
+      ]
     case "evaluate":
       return step.scripted
         ? [
@@ -558,6 +572,12 @@ export function buildHandoff(runDir: string, memoryDir?: string, opts: { replay?
       "",
     )
   }
+  if (steps.some((s) => s.op === "api")) {
+    md.push(
+      "Steps marked API are HTTP calls the run made in the browser's own session (its cookies went with each call, so a login and a cart carry over between the API and the UI steps). Make each one in the test the same way: through the framework's API request context that shares the browser's cookies (Playwright: page.request in TypeScript, Page.APIRequest in .NET, page.request in Python and Java), with the same method, URL, and body, reading the CSRF token where the step says (the page's form, or the csrftoken cookie) and sending it as the site expects. Assert the status the run got. Do not replace an API step with clicks, or a UI step with an API call.",
+      "",
+    )
+  }
   const slices = stepSlices(steps, opts.replay)
   if (slices.size) {
     md.push(

package/src/code/loop.ts

@@ -39,6 +39,7 @@ import {
   type AgentReport,
   type ExistingTest,
   type ReviewIssue,
+  type TestTarget,
 } from "./prompt.ts"
 import { openReviewer, type Reviewer } from "./review.ts"
 import { runTestCommand } from "./testrun.ts"
@@ -81,6 +82,44 @@ function attemptLine(attempt: number, outcome: string, detail: string, notes?: s
   return [`${attempt}: ${outcome}`, detail ? `: ${clip(detail.replace(/\s+/g, " ").trim(), 240)}` : "", notes ? ` (Scribe: ${clip(notes.replace(/\s+/g, " ").trim(), 240)})` : ""].join("")
 }
 
+/**
+ * Why a test failed, in up to three lines a person reads: the assertion or exception and its expected and received
+ * values, not the runner's summary ("Failed PracticePage_Flow [19 s]", "1 failed"). Knows dotnet test's "Error
+ * Message:" block, pytest's "E " lines, and the "Error:"/"Expected"/"Received" lines of Playwright, Jest, and JUnit.
+ */
+export function failureReason(output: string): string[] {
+  const lines = output.split("\n").map((l) => l.replace(/\x1b\[[0-9;]*m/g, "").trimEnd())
+  const tidy = (l: string) => l.trim().replace(/\s+/g, " ").slice(0, 300)
+  // dotnet test (NUnit, xUnit, MSTest): the lines between "Error Message:" and "Stack Trace:".
+  const message = lines.findIndex((l) => /^\s*Error Message:\s*$/.test(l))
+  if (message >= 0) {
+    const out: string[] = []
+    for (const l of lines.slice(message + 1)) {
+      if (/^\s*(Stack Trace:|Standard Output Messages:|Call log:)/.test(l)) break
+      if (l.trim()) out.push(tidy(l))
+      if (out.length === 3) break
+    }
+    if (out.length) return out
+  }
+  // pytest: the "E " lines under the failing line.
+  const pytest = lines.filter((l) => /^E\s{2,}\S/.test(l)).map((l) => tidy(l.slice(1)))
+  if (pytest.length) return pytest.slice(0, 3)
+  // Playwright, Jest, JUnit and the rest: the first error line and the expected and received values after it.
+  const summary = /^\s*(Failed\s+\S+\s+\[|Failed!|Total tests|\d+ (failed|passed)|Tests? (run|failed)|FAILED\b|=+ )/
+  // Source excerpts ("3 | expect(x).toBe(y)", "> 12 | …") quote the code, not the failure: they are passed over.
+  const excerpt = /^\s*>?\s*\d+\s*\|/
+  const first = lines.findIndex((l) => !summary.test(l) && !excerpt.test(l) && !/^\s*at\s/.test(l) && /(\w*(Error|Exception)\b[:\s]|expect\(|AssertionError|Assert\.|timed? ?out|Timeout \d+ms)/i.test(l))
+  if (first >= 0) {
+    const out = [tidy(lines[first]!)]
+    for (const l of lines.slice(first + 1, first + 12)) {
+      if (/^\s*(Expected|Received|Actual|But was|Call log)\b/i.test(l)) out.push(tidy(l))
+      if (out.length === 3) break
+    }
+    return out
+  }
+  return [firstError(output)].filter(Boolean)
+}
+
 /** The first line of a test run's output that names the error. */
 function firstError(output: string): string {
   const lines = output.split("\n").map((l) => l.trim()).filter(Boolean)
@@ -116,6 +155,8 @@ export async function runCodeAgent(opts: {
   knownDefect?: KnownDefect
   /** The test the case already has, which failed when ForgeQE ran it: the job updates it. */
   existingTest?: ExistingTest
+  /** An existing test chosen from measured coverage: reused as it is, or extended with what the run adds. */
+  target?: TestTarget
 }): Promise<CodeResult> {
   const given = path.resolve(opts.repo)
   if (!existsSync(given) || !statSync(given).isDirectory()) throw new Error(`Repo not found: ${given}`)
@@ -334,7 +375,7 @@ export async function runCodeAgent(opts: {
     let prompt =
       resumed && earlier?.stage === "writing"
         ? carryOnPrompt(liveOn)
-        : `${resumed ? `${RESUMED_NOTE}\n\n` : ""}${firstPrompt(handoff, { ...(live ? { liveBrowserDir: live.dir, liveOn } : {}), ...(known ? { notes: notesSection(known) } : {}), ...(profileText ? { profile: profileText } : {}), ...(opts.existingTest ? { existing: opts.existingTest } : {}) })}`
+        : `${resumed ? `${RESUMED_NOTE}\n\n` : ""}${firstPrompt(handoff, { ...(live ? { liveBrowserDir: live.dir, liveOn } : {}), ...(known ? { notes: notesSection(known) } : {}), ...(profileText ? { profile: profileText } : {}), ...(opts.existingTest ? { existing: opts.existingTest } : {}), ...(opts.target ? { target: opts.target } : {}) })}`
     if (prepared) {
       stage = "writing"
       saveSession()
@@ -495,7 +536,7 @@ export async function runCodeAgent(opts: {
         const used = now && changes ? outlines(now, classesUsedBy(now, changes)) : ""
         const settings = (repoProfile?.settings || []).map((st) => `- ${st.file}: ${Object.entries(st.values).map(([k, v]) => `${k}=${v}`).join(", ")}`)
         // After a rejection or a tidy-up, the reviewer checks its own earlier issues instead of starting over.
-        const verdict = await reviewer.review({ handoff, report, output: run.output, browsers: run.browsers, seconds: secs, changes, before: preexisting(before), ...(facts.length ? { facts } : {}), ...(used ? { outlines: used } : {}), ...(settings.length ? { settings } : {}), ...(result.review ? { earlier: result.review } : {}) })
+        const verdict = await reviewer.review({ handoff, report, output: run.output, browsers: run.browsers, seconds: secs, changes, before: preexisting(before), ...(facts.length ? { facts } : {}), ...(used ? { outlines: used } : {}), ...(settings.length ? { settings } : {}), ...(result.review ? { earlier: result.review } : {}), ...(opts.target ? { target: opts.target } : {}) })
         if (opts.signal?.aborted) {
           result.status = "stopped"
           break
@@ -528,6 +569,9 @@ export async function runCodeAgent(opts: {
             continue
           }
           result.status = "passed"
+          // What it did with the existing test it was given, for the result card and the agent.
+          if (opts.existingTest) result.existing = "updated"
+          else if (opts.target) result.existing = opts.target.mode === "reuse" ? "reused" : opts.target.mode === "update" ? "updated" : "extended"
           if (standards.length) {
             result.review = { ...verdict.review, approved: true, standardsLeft: true }
             result.notes = `The test passes and tests the goal. Coding standards still to fix: ${standards.map((i) => `${i.file ? `${i.file}: ` : ""}${i.problem}`).join(" ")}`
@@ -553,7 +597,24 @@ export async function runCodeAgent(opts: {
         finishWithKept(attempt, `the coding-standard fix made the test fail (${run.exitCode === null ? "timed out" : `exit ${run.exitCode}`})`)
         break
       }
+      // A test reused as it is fails: changing it changes checks someone relied on, so the user decides first.
+      if (opts.target?.mode === "reuse") {
+        const reason = failureReason(run.output)[0] || "no error message"
+        result.notes = `The existing test ${opts.target.test} (${opts.target.testFile}) covers this flow but fails now: ${reason}. Nothing was changed. Ask the user whether to update that test, or write a new one.`
+        emit({ type: "code", kind: "test", ok: false, attempt, text: `${report.test_file || opts.target.testFile} fails now: ${reason}; not changed, the user decides` })
+        break
+      }
       const why = run.exitCode === null ? `timed out after ${config.code.testTimeoutSeconds}s` : `exit ${run.exitCode}`
+      // ForgeQE's own replay of the run could not get through the flow either: no script reproduces it as recorded,
+      // so more attempts only spend tokens. Stop with what the replay met, for the user to act on.
+      const unreproducible = replay && !replay.error && replay.passes.length > 0 && replay.passes.every((p) => !p.reached)
+      if (unreproducible) {
+        const stuck = replay!.passes[0]!.steps.find((st) => !st.ok && !st.skipped)
+        const reason = failureReason(run.output)[0] || why
+        result.notes = `Stopped after attempt ${attempt}: the test failed (${reason}), and ForgeQE's replay of the run could not reach its final page either${stuck ? ` (step ${stuck.n}, ${stuck.action}: ${String(stuck.error || "").split("\n")[0]!.slice(0, 200)})` : ""}. The flow as recorded cannot be repeated by a script: usually a step the run did in a way a test cannot (a script in the page, state left from an earlier run, a pop-up only the agent dismissed). Run the case again after fixing that step, or check the handoff's replay section.`
+        emit({ type: "code", kind: "test", ok: false, attempt, text: `stopped: the flow cannot be reproduced; ForgeQE's replay stopped at ${stuck ? `step ${stuck.n}` : "a step"} as well (attempt ${attempt}/${maxAttempts})` })
+        break
+      }
       const mismatch = browserMismatch(runBrowser, run.browsers, report.command)
       if (mismatch) emit({ type: "code", kind: "info", text: `The test's browser differs from the run's: ${run.browsers.join(", ")}` })
       const key = failureKey(run.output)
@@ -565,16 +626,16 @@ export async function runCodeAgent(opts: {
         break
       }
       if (sameFailures === 1) {
-        emit({ type: "code", kind: "test", ok: false, attempt, text: `${report.test_file} failed the same way as the last attempt, ${why} (attempt ${attempt}/${maxAttempts})` })
+        emit({ type: "code", kind: "test", ok: false, attempt, text: `${report.test_file} failed the same way as the last attempt, ${why} (attempt ${attempt}/${maxAttempts}): ${failureReason(run.output)[0] || "no error message"}` })
         attemptLog.push(attemptLine(attempt, `failed, ${why}, the same way as before`, firstError(run.output), report.notes))
         nextTask = repeatedFailurePrompt(sameFailures + 1, report.command, run.exitCode, run.output, files) + mismatch + switchLiveOn()
-        next = { why: `the test failed the same way again (${why}); Scribe looks at the failing step first`, fixing: [firstError(run.output)].filter(Boolean) }
+        next = { why: `the test failed the same way again (${why}); Scribe looks at the failing step first`, fixing: failureReason(run.output) }
         continue
       }
-      emit({ type: "code", kind: "test", ok: false, attempt, text: `${report.test_file} failed, ${why} (attempt ${attempt}/${maxAttempts})` })
+      emit({ type: "code", kind: "test", ok: false, attempt, text: `${report.test_file} failed, ${why} (attempt ${attempt}/${maxAttempts}): ${failureReason(run.output)[0] || "no error message"}` })
       attemptLog.push(attemptLine(attempt, `failed, ${why}`, firstError(run.output), report.notes))
       nextTask = fixPrompt(attempt, report.command, run.exitCode, run.output, files) + mismatch
-      next = { why: `the test failed (${why})`, fixing: [firstError(run.output)].filter(Boolean) }
+      next = { why: `the test failed (${why})`, fixing: failureReason(run.output) }
     }
     keepSession = result.status === "stopped" || runtimeFailed
     if (report.test_name) result.testName = report.test_name

package/src/code/notes.ts

@@ -73,3 +73,79 @@ export function notesSection(n: RepoNotes): string {
   lines.push("", "Start from these notes instead of reading the repository again: read only the files this test uses or changes, and read a file before you edit or reuse it, since it may have changed.")
   return lines.join("\n")
 }
+
+/**
+ * Words that say how a goal is phrased, not what flow it is: "verify each value is accepted and remains present" and
+ * "verify the values are accepted" ask for the same thing. Left out when two goals are compared.
+ */
+const FILLER = new Set(
+  "a an and the of in on to with within only each every any all then this that is are be as it its if for from by or not when also into page step steps result results report run test flow treat fail verify verifying check checks expected appears appear present remains accepted"
+    .split(" "),
+)
+
+/** A goal's content words, endings off ("toggles", "toggled" are one), for telling the same flow from another. */
+function goalWords(goal: string): Set<string> {
+  return new Set(
+    goal
+      .toLowerCase()
+      .replace(/https?:\/\/(www\.)?/g, "")
+      .split(/[^a-z0-9.@-]+/)
+      .map((w) => w.replace(/^[.-]+|[.-]+$/g, ""))
+      .filter((w) => w.length > 1 && !FILLER.has(w))
+      .map((w) => w.replace(/(ing|ed|es|s)$/, "")),
+  )
+}
+
+/** The website a goal names, if any: the same steps on another site are another flow. */
+function goalHost(goal: string): string {
+  return /https?:\/\/(?:www\.)?([^/\s,)]+)/i.exec(goal)?.[1]?.toLowerCase() || ""
+}
+
+/**
+ * How alike two goals are, 0 to 1: the content words they share, over the words they have (Dice). Measured on real
+ * goals ForgeQE's agent wrote: the same flow reworded scores 0.88; a shorter flow on the same page 0.59, and the same
+ * flow with a different final check 0.67.
+ */
+export function goalSimilarity(a: string, b: string): number {
+  const [ha, hb] = [goalHost(a), goalHost(b)]
+  if (ha && hb && ha !== hb) return 0
+  const x = goalWords(a)
+  const y = goalWords(b)
+  if (!x.size || !y.size) return 0
+  let shared = 0
+  for (const w of x) if (y.has(w)) shared++
+  return (2 * shared) / (x.size + y.size)
+}
+
+/** Goals this alike are the same flow asked again; ForgeQE's agent rewords a goal from one request to the next. */
+export const SAME_GOAL = 0.8
+
+/**
+ * Whether `goal` asks again for the flow an earlier test was written for. Rewording adds words and keeps the earlier
+ * ones; another flow drops what the earlier goal checked ("pie chart visible" becoming "list can be reordered"), so at
+ * most one of the earlier goal's content words may be missing, and the two must be alike overall.
+ */
+export function sameFlow(earlier: string, goal: string): boolean {
+  if (goalSimilarity(earlier, goal) < SAME_GOAL) return false
+  const now = goalWords(goal)
+  let dropped = 0
+  for (const w of goalWords(earlier)) if (!now.has(w)) dropped++
+  return dropped <= 1
+}
+
+/**
+ * The test an earlier job wrote in this repository for the same goal, when its file is still there: the newest of the
+ * closest matches. Notes of any age count here, since the test file itself is what is checked.
+ */
+export function findWrittenTest(memoryDir: string, repo: string, goal: string, exists: (file: string) => boolean): WrittenTest | null {
+  const tests = load(memoryDir).repos[repo]?.tests || []
+  let best: { test: WrittenTest; score: number } | null = null
+  for (const t of tests) {
+    const score = goalSimilarity(t.goal, goal)
+    if (sameFlow(t.goal, goal) && exists(t.test_file) && (!best || score > best.score || (score === best.score && t.at > best.test.at))) best = { test: t, score }
+  }
+  return best?.test || null
+}
+
+/** Where browser_run leaves the earlier test for a goal that failed, so write_test updates it instead of adding one. */
+export const EXISTING_TEST_FILE = "existing-test.json"

package/src/code/prompt.ts

@@ -74,6 +74,49 @@ export function preparePrompt(goal: string, checking = true): string {
 /** A test this case already has, which now fails: the job updates it instead of writing a new one. */
 export type ExistingTest = { testFile: string; command: string; output: string; why: string }
 
+/**
+ * An existing test the job works from, chosen from measured coverage (or by the person): `reuse` it as it is, or
+ * `extend` it with only the steps and checks it lacks.
+ */
+export type TestTarget = { mode: "reuse" | "extend" | "update"; testFile: string; test: string; line: number; covered: string[]; missing: string[]; why: string }
+
+function targetSection(t: TestTarget): string[] {
+  const list = (items: string[]) => items.map((i) => `- ${i}`)
+  if (t.mode === "update") {
+    return [
+      "## Update the existing test, do not write a new one",
+      "",
+      `The user asked to update ${t.test} (${t.testFile}:${t.line}) so it matches the verified run below. Change only what no longer matches the run (a step, a locator, an expected value); keep its other steps and assertions. Do not create a new test file.`,
+      "Do not change what an existing page-object member or method does for the other tests that use it (code_references shows them); add a member instead, or keep its behaviour for them, and say so in notes.",
+      "",
+    ]
+  }
+  if (t.mode === "reuse") {
+    return [
+      "## The existing test that already covers this flow",
+      "",
+      `ForgeQE's code graph found that ${t.test} (${t.testFile}:${t.line}) already reaches every element of this run (${t.why}). Do not write or change any test or page object.`,
+      "Find the command that runs only that test (from your notes on this repository, or the repository's test setup), and give the structured result with that test_file, test_name, and command. ForgeQE runs it. If it fails, ForgeQE stops and asks the user before anything is changed.",
+      "",
+    ]
+  }
+  return [
+    "## Extend the existing test, do not write a new one",
+    "",
+    `ForgeQE's code graph found that ${t.test} (${t.testFile}:${t.line}) already covers most of this flow (${t.why}). Extend that test: add only what it lacks, in the order of the flow, through the same page objects and helpers. Keep every step and assertion it has. Do not create a new test file.`,
+    "",
+    "It already covers:",
+    ...list(t.covered),
+    "",
+    "Add only these:",
+    ...list(t.missing),
+    "",
+    "When the missing steps start from a different place than the test does, add one test method next to it in the same class, reusing its setup and helpers, instead of a new file.",
+    "Add page-object members for elements the repository does not locate yet. Do not change what an existing member or method does: other tests use it (code_references shows which). When one must change, keep its behaviour for those tests and say so in notes.",
+    "",
+  ]
+}
+
 function existingSection(e: ExistingTest): string[] {
   return [
     "## The test this case already has",
@@ -91,7 +134,7 @@ function existingSection(e: ExistingTest): string[] {
   ]
 }
 
-export function firstPrompt(handoff: Handoff, opts: { liveBrowserDir?: string; liveOn?: boolean; notes?: string; profile?: string; existing?: ExistingTest } = {}): string {
+export function firstPrompt(handoff: Handoff, opts: { liveBrowserDir?: string; liveOn?: boolean; notes?: string; profile?: string; existing?: ExistingTest; target?: TestTarget } = {}): string {
   const live = opts.liveBrowserDir
     ? opts.liveOn
       ? `Your live_browser_* tools are on because the replay failed at a step; use them on that step's page. They save snapshots and screenshots in ${opts.liveBrowserDir}.`
@@ -100,9 +143,15 @@ export function firstPrompt(handoff: Handoff, opts: { liveBrowserDir?: string; l
   return [
     opts.existing
       ? `Update the existing test for the flow below so it passes again. When it is updated, give the structured result; ForgeQE then runs it.`
-      : `Write a test in this repository for the flow below, reusing the repository's existing test code. When it is written, give the structured result; ForgeQE then runs it.`,
+      : opts.target?.mode === "reuse"
+        ? `An existing test already covers the flow below. Report how to run it; change nothing. ForgeQE then runs it.`
+        : opts.target?.mode === "extend"
+          ? `Extend the existing test named below with what the flow below adds. When it is extended, give the structured result; ForgeQE then runs it.`
+          : opts.target?.mode === "update"
+            ? `Update the existing test named below to match the flow below. When it is updated, give the structured result; ForgeQE then runs it.`
+          : `Write a test in this repository for the flow below, reusing the repository's existing test code. When it is written, give the structured result; ForgeQE then runs it.`,
     "",
-    ...(opts.existing ? existingSection(opts.existing) : []),
+    ...(opts.existing ? existingSection(opts.existing) : opts.target ? targetSection(opts.target) : []),
     ...(opts.profile ? [opts.profile, ""] : []),
     ...(opts.notes ? [opts.notes, ""] : []),
     handoff.markdown,

package/src/code/replay.ts

@@ -135,11 +135,31 @@ function describe(step: Step): string {
       return "go back"
     case "evaluate":
       return "run the agent's JavaScript"
+    case "api":
+      return `API ${step.method || "GET"} ${step.url}`
     default:
       return `${step.op || "unknown"}${step.action_keys?.length ? ` (${step.action_keys.join(", ")})` : ""}`
   }
 }
 
+/**
+ * An API step as the engine's api_request makes it (py/forgeqe_engine/api_step.py): fetch in the page with its cookies,
+ * the CSRF token from the cookie or the page's form, sent in headers and, for a form post, in the form.
+ */
+export const API_CALL_JS = `async (a) => {
+  const cookie = (n) => { const c = document.cookie.split('; ').find((x) => x.startsWith(n + '=')); return c ? decodeURIComponent(c.slice(n.length + 1)) : '' };
+  let token = '';
+  if (a.csrf === 'cookie') token = cookie('csrftoken') || cookie('XSRF-TOKEN') || cookie('csrf_token') || cookie('_csrf');
+  if (a.csrf === 'form') { const f = document.querySelector('input[name="csrfmiddlewaretoken"], input[name="_csrf"], input[name="csrf_token"], input[name="authenticity_token"], input[name="__RequestVerificationToken"]'); const m = document.querySelector('meta[name="csrf-token"], meta[name="_csrf"]'); token = (f && f.value) || (m && m.content) || ''; }
+  const headers = {};
+  if (a.contentType) headers['Content-Type'] = a.contentType;
+  let body = a.body || undefined;
+  if (token) { headers['X-CSRFToken'] = token; headers['X-XSRF-TOKEN'] = token; headers['X-CSRF-Token'] = token; if (/x-www-form-urlencoded/i.test(a.contentType || '') && !/(^|&)(csrfmiddlewaretoken|_csrf|authenticity_token)=/.test(body || '')) body = (body ? body + '&' : '') + 'csrfmiddlewaretoken=' + encodeURIComponent(token); }
+  const method = (a.method || 'GET').toUpperCase();
+  try { const r = await fetch(a.url, { method, headers, body: method === 'GET' || method === 'HEAD' ? undefined : body, credentials: 'include', redirect: 'follow' }); return { status: r.status, final_url: r.url }; }
+  catch (e) { return { error: String(e && e.message || e) }; }
+}`
+
 /** The body of one step in the script, or null when it has no Playwright equivalent. */
 function stepCode(step: Step): string | null {
   const loc = step.locators?.map(locatorExpr).find(Boolean) || null
@@ -171,6 +191,19 @@ function stepCode(step: Step): string | null {
       return "(page) => page.goBack()"
     case "evaluate":
       return step.code ? `(page) => page.evaluate(${q(step.code)})` : null
+    case "api": {
+      if (!step.url) return null
+      // The call is made in the page, as the run made it: the session's cookies and its CSRF token go with it. The step
+      // fails when the server answers differently than it did in the run (403 where the run got 200).
+      const args = JSON.stringify({ method: step.method || "GET", url: step.url, body: step.body || "", contentType: step.content_type || "", csrf: step.csrf || "" })
+      return [
+        "async (page) => {",
+        `    const r = await page.evaluate(${API_CALL_JS}, ${args})`,
+        "    if (r.error) throw new Error(`API ${r.error}`)",
+        ...(step.status ? [`    if (r.status !== ${Number(step.status)}) throw new Error(\`API answered \${r.status}, the run got ${Number(step.status)}\`)`] : []),
+        "  }",
+      ].join("\n")
+    }
     default:
       return null
   }