@capagents/forgeqe 1.1.4 → 1.1.5

package/src/cases/rca.ts

@@ -0,0 +1,258 @@
+import { spawnSync } from "node:child_process"
+import { existsSync } from "node:fs"
+import path from "node:path"
+
+import type { AppConfig } from "../config.ts"
+import { openSession, type AgentSpec, type OpencodeSession } from "../code/runtime.ts"
+import { privateEnvNames } from "../childEnv.ts"
+import { domainNotes } from "../domain.ts"
+import { prompt } from "../prompts.ts"
+import type { CodeEvent, Usage } from "../types.ts"
+import { costOf, type Cost } from "../usage.ts"
+import { lastPassed } from "./defectHistory.ts"
+import { routeOf, type DefectCluster } from "./defects.ts"
+import { reproText } from "./repro.ts"
+import type { CaseResult } from "./types.ts"
+
+/**
+ * Defect intelligence, part three (b): why a defect may happen. Always a hypothesis, never a verdict.
+ *
+ * - From the evidence alone, at no cost: the layer it most likely sits in (the server, the front end, access, the
+ *   environment) and the component (the route, the script).
+ * - From the product's repository (defects.product_repo): suspect commits, made since the defect's cases last
+ *   passed, whose changes mention its route or error (git log -G).
+ * - From a model, for new defects and regressions only (defects.analyze): a read-only agent reads the evidence, the
+ *   suspect commits, and the product's code, and writes a hypothesis with its confidence and how to confirm it.
+ */
+
+export type Suspect = { sha: string; subject: string; author: string; at: string; files: string[]; why: string }
+
+export type RootCause = {
+  layer: string
+  component?: string
+  /** What the layer was read from. */
+  basis: string
+  suspects?: Suspect[]
+  /** The commits searched, e.g. "since 2026-10-12, when SHOP-1 last passed". */
+  range?: string
+  hypothesis?: string
+  confidence?: "low" | "medium" | "high"
+  evidence?: string[]
+  confirm?: string[]
+  files?: string[]
+  /** Why there is no hypothesis or no suspect commits, when there is a reason to give. */
+  note?: string
+  model?: string
+  tokens?: number
+  usage?: Usage
+  cost?: Cost
+}
+
+const STATUS = /returns (\d{3})/
+
+/** The layer and component a defect points to, from its evidence. */
+export function layerOf(c: DefectCluster): Pick<RootCause, "layer" | "component" | "basis"> {
+  if (c.kind === "request") {
+    const status = Number(STATUS.exec(c.title)?.[1] || 0)
+    const component = c.title.replace(/ (returns \d{3}|fails)$/, "")
+    if (status >= 500) return { layer: "Server (API)", component, basis: `the server answered ${status}` }
+    if (status === 401 || status === 403) return { layer: "Access: sign-in or permissions", component, basis: `the server refused with ${status}` }
+    if (status === 404) return { layer: "API route or deployment", component, basis: "the route was not found" }
+    if (status >= 400) return { layer: "API request: the page sent what the server refused", component, basis: `the server answered ${status}` }
+    return { layer: "Network", component, basis: "the request did not complete" }
+  }
+  if (c.kind === "exception") {
+    const at = / at (\S+?)(?::\d+)?$/.exec(c.evidence[0] || "")?.[1]
+    return { layer: "Front end (JavaScript)", ...(at ? { component: routeOf(at) } : {}), basis: "an uncaught error in the page" }
+  }
+  if (c.kind === "console") return { layer: "Front end", ...(c.page ? { component: routeOf(c.page) } : {}), basis: "an error in the browser console" }
+  const page = c.page ? { component: routeOf(c.page) } : {}
+  if (c.failure === "environment") return { layer: "Environment", ...page, basis: "the run met an environment problem" }
+  if (c.failure === "test_case_issue") return { layer: "Test case", ...page, basis: "the test case does not fit the application" }
+  if (c.failure === "automation") return { layer: "Test automation", ...page, basis: "the browser agent could not finish" }
+  return { layer: "Application behaviour (a page or business rule)", ...page, basis: "the page showed a wrong result with no browser error" }
+}
+
+const escape = (s: string) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")
+const GENERIC = new Set(["api", "v1", "v2", "v3", "rest", "graphql", "www", "app", "index", "home"])
+
+/** What a change that caused it would mention: the route and its words, or the error message. */
+export function searchTerms(c: DefectCluster): string[] {
+  if (c.kind === "request" && c.term) return [c.term, ...c.term.split("/").filter((w) => w.length >= 3 && !GENERIC.has(w.toLowerCase()))]
+  if (c.kind === "exception" && c.term) return [c.term, ...c.term.split(/\W+/).filter((w) => w.length >= 5).slice(0, 3)]
+  if (c.page) {
+    const p = routeOf(c.page)
+    return p.slice(p.indexOf("/")).split("/").filter((w) => w.length >= 3 && !GENERIC.has(w.toLowerCase()))
+  }
+  return []
+}
+
+const git = (repo: string, args: string[]) => spawnSync("git", ["-C", repo, ...args], { encoding: "utf8", maxBuffer: 20 * 1024 * 1024 })
+
+type Commit = { sha: string; subject: string; author: string; at: string; files: string[] }
+
+function commits(repo: string, args: string[]): Commit[] {
+  const out = git(repo, ["log", "--no-merges", "-n", "300", "--format=%x1e%H%x1f%an%x1f%aI%x1f%s", "--name-only", ...args])
+  if (out.status !== 0) return []
+  return out.stdout
+    .split("\x1e")
+    .filter((b) => b.trim())
+    .map((b) => {
+      const [head, ...files] = b.trim().split("\n")
+      const [sha, author, at, subject] = head.split("\x1f")
+      return { sha, author, at, subject, files: files.map((f) => f.trim()).filter(Boolean) }
+    })
+}
+
+/**
+ * Commits since `since` whose changes mention the defect's terms (strongest first), or whose files are named after
+ * them. Five at most; none when the folder is not a git repository.
+ */
+export function suspectCommits(repo: string, c: DefectCluster, since: string): Suspect[] {
+  const terms = searchTerms(c)
+  if (!terms.length || !existsSync(path.join(repo, ".git"))) return []
+  const changed = commits(repo, [`--since=${since}`, "-i", `-G${terms.map(escape).join("|")}`])
+  const all = commits(repo, [`--since=${since}`])
+  const scored = new Map<string, { c: Commit; score: number; why: string[] }>()
+  for (const x of changed) scored.set(x.sha, { c: x, score: 2, why: [`its changes mention ${terms.slice(0, 2).map((t) => `\`${t}\``).join(" or ")}`] })
+  for (const x of all) {
+    const named = x.files.filter((f) => terms.slice(1).some((t) => f.toLowerCase().includes(t.toLowerCase())))
+    if (!named.length) continue
+    const s = scored.get(x.sha) || { c: x, score: 0, why: [] }
+    s.score += 1
+    s.why.push(`it changes ${named.slice(0, 2).map((f) => `\`${f}\``).join(", ")}`)
+    scored.set(x.sha, s)
+  }
+  return [...scored.values()]
+    .sort((a, b) => b.score - a.score || b.c.at.localeCompare(a.c.at))
+    .slice(0, 5)
+    .map(({ c: x, why }) => ({ sha: x.sha, subject: x.subject, author: x.author, at: x.at, files: x.files.slice(0, 8), why: why.join("; ") }))
+}
+
+/** The layer, and the suspect commits when the product's repository is known: no model, no cost. */
+export function basicCause(c: DefectCluster, opts: { suiteDir: string; productRepo?: string; now?: string }): RootCause {
+  const rc: RootCause = { ...layerOf(c) }
+  if (!opts.productRepo) return rc
+  const passed = lastPassed(opts.suiteDir, c.cases.map((x) => x.id))
+  if (!passed) {
+    rc.note = "No earlier run where its cases passed: no commit range to search."
+    return rc
+  }
+  rc.range = `since ${passed.slice(0, 16).replace("T", " ")}, the last run where its cases passed`
+  rc.suspects = suspectCommits(opts.productRepo, c, passed)
+  return rc
+}
+
+export const rcaAgentPrompt = () => prompt("defect-rca")
+
+export const RCA_AGENT: AgentSpec = {
+  name: "forgeqe-defect-rca",
+  description: "Writes a root-cause hypothesis for a defect from its evidence and the product's code",
+  get prompt() {
+    return rcaAgentPrompt()
+  },
+  permission: {
+    edit: "deny",
+    webfetch: "deny",
+    task: "deny",
+    "live_*": "deny",
+    bash: { "*": "deny", "git log*": "allow", "git show*": "allow", "git diff*": "allow", "git blame*": "allow", "git ls-files*": "allow", "git grep*": "allow", "ls*": "allow" },
+  },
+  steps: 25,
+}
+
+const lines = { type: "array", items: { type: "string" } } as const
+
+export const RCA_SCHEMA = {
+  type: "object",
+  properties: {
+    hypothesis: { type: "string", description: "The most likely cause, in two sentences or fewer" },
+    confidence: { type: "string", enum: ["low", "medium", "high"] },
+    layer: { type: "string", description: "Where the cause most likely sits, e.g. Server (API), Front end, Data, Configuration" },
+    component: { type: "string", description: "The file, module, route, or service, as named in the code or the evidence" },
+    evidence: { ...lines, description: "The facts the hypothesis rests on, each from the evidence or a file you read, with where" },
+    confirm: { ...lines, description: "Checks a developer can make to confirm or reject it" },
+    files: { ...lines, description: "Repository paths of the code to look at first" },
+  },
+  required: ["hypothesis", "confidence", "layer", "evidence", "confirm", "files"],
+} as const
+
+type RcaReport = { hypothesis?: string; confidence?: string; layer?: string; component?: string; evidence?: string[]; confirm?: string[]; files?: string[] }
+
+/** What the agent is told: the defect, how it happened, its history, and the suspect commits. */
+export function rcaPrompt(c: DefectCluster, first: CaseResult, rc: RootCause, productRepo?: string): string {
+  return [
+    `Defect: ${c.title}`,
+    `Kind of failure: ${c.failureLabel}. It breaks ${c.cases.length} case${c.cases.length === 1 ? "" : "s"}: ${c.cases.map((x) => `${x.id} ${x.title}`).join("; ")}.`,
+    "",
+    "Evidence from the runs:",
+    ...c.evidence.map((e) => `- ${e}`),
+    ...(c.pages.length ? [`Pages: ${c.pages.join(", ")}`] : []),
+    "",
+    `What the browser agent reported for ${first.case.id}: ${first.summary.split("\n")[0]}`,
+    ...(c.repro ? ["", reproText(c.repro)] : []),
+    "",
+    `From the evidence alone: layer ${rc.layer}${rc.component ? `, component ${rc.component}` : ""} (${rc.basis}).`,
+    ...(c.history ? [`History: ${c.history.status}${c.history.seen ? `, seen in ${c.history.seen} earlier runs` : ""}${c.history.previousBug ? `, earlier bug ${c.history.previousBug.label}` : ""}.`] : []),
+    ...(rc.suspects?.length ? ["", `Suspect commits (${rc.range}):`, ...rc.suspects.map((s) => `- ${s.sha.slice(0, 10)} ${s.subject} (${s.author}, ${s.at.slice(0, 10)}): ${s.why}. Files: ${s.files.join(", ")}`)] : rc.range ? ["", `No commit ${rc.range} mentions it.`] : []),
+    "",
+    productRepo ? `The product's code is in ${productRepo}. Read it.` : "You have no access to the product's code. Reason from the evidence only, and say so in the confidence.",
+  ].join("\n")
+}
+
+/**
+ * The model's hypothesis for one defect, added to `rc`. A failure leaves `rc` with what it had and a note: the
+ * analysis is a hint, never a reason for a suite to fail.
+ */
+export async function analyzeCause(
+  config: AppConfig,
+  c: DefectCluster,
+  first: CaseResult,
+  rc: RootCause,
+  opts: { cwd: string; productRepo?: string; signal?: AbortSignal; onEvent?: (e: CodeEvent) => void; open?: (o: Parameters<typeof openSession>[0]) => Promise<OpencodeSession<RcaReport>> },
+): Promise<RootCause> {
+  const repo = opts.productRepo || opts.cwd
+  const open = (opts.open || openSession) as (o: Parameters<typeof openSession>[0]) => Promise<OpencodeSession<RcaReport>>
+  let session: OpencodeSession<RcaReport> | undefined
+  try {
+    session = await open({
+      repo,
+      profile: config.llm,
+      domain: domainNotes(config),
+      model: config.code.model,
+      bin: config.code.opencode,
+      readDirs: [repo],
+      hide: privateEnvNames(config),
+      signal: opts.signal,
+      onEvent: opts.onEvent,
+      agent: RCA_AGENT,
+      schema: RCA_SCHEMA as unknown as Record<string, unknown>,
+      title: `ForgeQE root cause: ${c.title}`.slice(0, 120),
+      role: "root-cause analyst",
+    })
+    let reply = await session.prompt(rcaPrompt(c, first, rc, opts.productRepo))
+    if (!reply.report && !reply.error && !opts.signal?.aborted) reply = await session.prompt("Give the structured result: hypothesis, confidence, layer, component, evidence, confirm, and files.")
+    const usage: Usage = { ...session.usage }
+    const spent = { model: session.model, tokens: usage.input + usage.output, usage, ...(costOf(usage, config.llm.price) ? { cost: costOf(usage, config.llm.price) } : {}) }
+    const r = reply.report
+    if (!r?.hypothesis) return { ...rc, ...spent, note: `No hypothesis${reply.error ? `: ${reply.error}` : "."}` }
+    const confidence = r.confidence === "high" || r.confidence === "medium" ? r.confidence : "low"
+    // A guess does not replace what the evidence says: the model's layer and component only with code behind them.
+    const sure = confidence !== "low"
+    return {
+      ...rc,
+      ...(sure && r.layer ? { layer: r.layer } : {}),
+      ...(sure && r.component ? { component: r.component } : {}),
+      hypothesis: r.hypothesis.trim(),
+      confidence,
+      evidence: (r.evidence || []).map(String).filter(Boolean).slice(0, 8),
+      confirm: (r.confirm || []).map(String).filter(Boolean).slice(0, 5),
+      files: (r.files || []).map(String).filter(Boolean).slice(0, 8),
+      ...spent,
+    }
+  } catch (err) {
+    return { ...rc, note: `No hypothesis: ${(err as Error).message.split("\n")[0].slice(0, 200)}` }
+  } finally {
+    await session?.close()
+  }
+}

package/src/cases/readiness.ts

@@ -68,6 +68,38 @@ export function verdictOf(r: Pick<Readiness, "stories" | "bugs" | "counts" | "pa
   return { verdict: blockers.length ? "NO-GO" : risks.length ? "GO WITH RISKS" : "GO", blockers, risks }
 }
 
+type SavedDefect = { title: string; failureLabel?: string; cases: { id: string }[]; bug?: { label: string }; history?: { status?: string; previousBug?: { label: string } } }
+
+/**
+ * What the newest suite with defects says about the test cases in scope (Azure DevOps ids): each regression is a
+ * blocker, each new defect without a bug a risk. Cases are matched by their work item (ADO-123, or the id in ref).
+ */
+export function defectSignals(suitesDir: string, inScope: Set<number>): { blockers: string[]; risks: string[] } {
+  const out = { blockers: [] as string[], risks: [] as string[] }
+  if (!existsSync(suitesDir) || !inScope.size) return out
+  const dirs = readdirSync(suitesDir).filter((d) => existsSync(path.join(suitesDir, d, "suite.json"))).sort().reverse()
+  for (const dir of dirs) {
+    let suite: { results?: { case: { id: string; ref?: { workItem?: number } } }[]; defects?: SavedDefect[] }
+    try {
+      suite = JSON.parse(readFileSync(path.join(suitesDir, dir, "suite.json"), "utf8"))
+    } catch {
+      continue
+    }
+    if (!Array.isArray(suite.defects)) continue
+    const workItem = new Map((suite.results || []).map((r) => [r.case.id, Number(r.case.ref?.workItem || /^ADO-(\d+)/.exec(r.case.id)?.[1] || 0)]))
+    const ids = (d: SavedDefect) => d.cases.map((c) => workItem.get(c.id) || workItem.get(c.id.split("@")[0]) || Number(/^ADO-(\d+)/.exec(c.id)?.[1] || 0)).filter((n) => inScope.has(n))
+    for (const d of suite.defects) {
+      const hit = ids(d)
+      if (!hit.length) continue
+      const cases = `test case${hit.length === 1 ? "" : "s"} ${[...new Set(hit)].join(", ")}`
+      if (d.history?.status === "regression") out.blockers.push(`regression: ${d.title} came back${d.history.previousBug ? ` after ${d.history.previousBug.label}` : ""} (${cases})${d.bug ? `, ${d.bug.label}` : ""}`)
+      else if (d.history?.status === "new" && !d.bug) out.risks.push(`new defect with no bug: ${d.title} (${cases})`)
+    }
+    break
+  }
+  return out
+}
+
 /** The newest result per test case in ForgeQE's suite reports. */
 function forgeqeResults(suitesDir: string): Map<number, { outcome: Outcome; when: string }> {
   const out = new Map<number, { outcome: Outcome; when: string }>()
@@ -228,7 +260,12 @@ export async function releaseReadiness(cfg: AppConfig, repo: string, scope: Scop
   }
   const passRate = executed.length ? Math.round((passed / executed.length) * 1000) / 10 : null
   const v = verdictOf({ stories, bugs, counts, passRate }, thresholds)
-  return { scope: label, stories, bugs, counts, passRate, ...v, notes }
+  // ForgeQE's latest suite: a regression on a case in scope blocks the release; a new defect without a bug is a risk.
+  const inScope = new Set(stories.flatMap((st) => st.cases.map((c) => c.id)))
+  const d = defectSignals(cfg.suite.outDir, inScope)
+  const blockers = [...v.blockers, ...d.blockers]
+  const risks = [...v.risks, ...d.risks]
+  return { scope: label, stories, bugs, counts, passRate, verdict: blockers.length ? "NO-GO" : risks.length ? "GO WITH RISKS" : "GO", blockers, risks, notes }
 }
 
 const MARK: Record<Outcome, string> = { passed: "✅ passed", failed: "❌ failed", blocked: "⛔ blocked", "not run": "⚪ not run" }

package/src/cases/repro.ts

@@ -0,0 +1,202 @@
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"
+import path from "node:path"
+
+import type { ReportEnv } from "../report/html.ts"
+import { readTimeline } from "../report/data.ts"
+import { routeOf, type CaseDefect, type DefectCluster } from "./defects.ts"
+import { stepVerdictLines, type CaseResult } from "./types.ts"
+
+/**
+ * Defect intelligence, part three (a): how to make a defect happen again. Built from what the run of its first case
+ * recorded, with nothing new sent anywhere: the case's steps with the agent's verdicts, what the browser did, where
+ * and with what it ran, and the failing request as a curl command. The engine has already taken credentials out of
+ * the requests it records (py/forgeqe_engine/network.py), so the command carries no secret.
+ */
+
+export type Reproducibility = "always" | "intermittent" | "once"
+
+export type FailingRequest = { method: string; url: string; status?: number; requestBody?: string; responseBody?: string; curl: string }
+
+export type ReproPack = {
+  reproducibility: Reproducibility
+  /** Why: "failed the same way in 2 of 2 runs", "seen in 3 earlier runs". */
+  detail: string
+  /** The case's steps, with the agent's verdict on each when it gave one. */
+  steps: string[]
+  /** What the browser did before it failed: the actions that worked, in order. */
+  actions: string[]
+  startUrl?: string
+  endUrl?: string
+  /** When the run ended (ISO): the report shows it in the reader's time; the env line gives it in UTC. */
+  at: string
+  env: string[]
+  request?: FailingRequest
+  /** Files from the run that help: its folder relative to the suite folder, and each file. */
+  files: { name: string; path: string }[]
+  /** The repro file written for the defect, relative to the suite folder. */
+  file?: string
+}
+
+type Call = { method?: string; url?: string; request?: { headers?: Record<string, string>; body?: string }; response?: { status?: number; body?: string } }
+
+const MAX_ACTIONS = 15
+const MAX_BODY = 600
+
+const clip = (s: string, n: number) => (s.length > n ? `${s.slice(0, n - 1)}…` : s)
+const quote = (s: string) => `'${s.replace(/'/g, `'\\''`)}'`
+
+/** The request a "request" defect is about, from the run's network.json: the last call to its route with its status. */
+export function failingRequest(runDir: string | undefined, d: CaseDefect): FailingRequest | undefined {
+  if (d.kind !== "request" || !runDir) return undefined
+  const file = path.join(runDir, "network.json")
+  if (!existsSync(file)) return undefined
+  let calls: Call[] = []
+  try {
+    calls = (JSON.parse(readFileSync(file, "utf8")) as { calls?: Call[] }).calls || []
+  } catch {
+    return undefined
+  }
+  const hit = [...calls].reverse().find((c) => c.url && (c.response?.status || 0) >= 400 && d.title.startsWith(`${routeOf(c.url)} `))
+  if (!hit?.url) return undefined
+  const method = (hit.method || "GET").toUpperCase()
+  const headers = Object.entries(hit.request?.headers || {}).map(([k, v]) => `-H ${quote(`${k}: ${v}`)}`)
+  const body = hit.request?.body
+  const curl = ["curl -i", ...(method === "GET" ? [] : [`-X ${method}`]), quote(hit.url), ...headers, ...(body ? [`--data-raw ${quote(body)}`] : [])].join(" ")
+  return {
+    method,
+    url: hit.url,
+    ...(hit.response?.status ? { status: hit.response.status } : {}),
+    ...(body ? { requestBody: clip(body, MAX_BODY) } : {}),
+    ...(hit.response?.body ? { responseBody: clip(hit.response.body, MAX_BODY) } : {}),
+    curl,
+  }
+}
+
+/** The browser's actions that worked, without repeats in a row and without the agent's own bookkeeping. */
+function browserActions(runDir?: string): string[] {
+  const out: string[] = []
+  for (const t of readTimeline(runDir)) {
+    for (const a of t.actions || []) {
+      if (!a.ok || a.done || !a.text.trim() || /^(done|wait\b)/i.test(a.text)) continue
+      if (out[out.length - 1] !== a.text) out.push(a.text)
+    }
+  }
+  return out.length > MAX_ACTIONS ? [...out.slice(0, MAX_ACTIONS - 1), `… and ${out.length - MAX_ACTIONS + 1} more (steps.json)`] : out
+}
+
+/** How sure it is that the defect happens again: a second run of it, else its history, else seen once. */
+export function reproducibility(r: CaseResult, seen = 0, intermittent?: { failed: number; runs: number }): { reproducibility: Reproducibility; detail: string } {
+  if (intermittent) return { reproducibility: "intermittent", detail: `its case failed in ${intermittent.failed} of the last ${intermittent.runs} runs` }
+  if (r.reproduced) {
+    const { runs, failed } = r.reproduced
+    return failed >= runs
+      ? { reproducibility: "always", detail: `failed the same way in ${failed} of ${runs} runs` }
+      : { reproducibility: "intermittent", detail: `failed the same way in ${failed} of ${runs} runs` }
+  }
+  if (seen) return { reproducibility: "always", detail: `seen in ${seen} earlier run${seen === 1 ? "" : "s"} too` }
+  return { reproducibility: "once", detail: "seen once; not run again" }
+}
+
+/** The repro pack of a defect, from the run of one of its cases (the first). `suiteDir` makes file paths relative. */
+export function reproPack(r: CaseResult, d: CaseDefect, opts: { env?: ReportEnv; seen?: number; intermittent?: { failed: number; runs: number }; suiteDir?: string } = {}): ReproPack {
+  const verdicts = stepVerdictLines(r)
+  const steps = verdicts.length ? verdicts : r.case.steps.map((s, i) => `${i + 1}. ${s.action.trim()}${s.data ? ` (data: ${s.data})` : ""}${s.expected ? ` — expected: ${s.expected}` : ""}`)
+  const e = opts.env
+  const env = [
+    ...(e?.browser ? [`Browser: ${e.browser}${r.device ? `, device ${r.device}` : ""}`] : r.device ? [`Device: ${r.device}`] : []),
+    ...(e?.ci ? [`Build: ${e.ci.name}${e.ci.url ? ` ${e.ci.url}` : ""}`] : []),
+    ...(e?.commit ? [`Commit: ${e.commit}${e.branch ? ` on ${e.branch}` : ""}`] : []),
+    ...(e?.version ? [`ForgeQE ${e.version} on ${e.machine}`] : []),
+    `When: ${r.finishedAt.slice(0, 16).replace("T", " ")} UTC`,
+  ]
+  const rel = (f: string) => (opts.suiteDir ? path.relative(opts.suiteDir, f) : f)
+  const files: ReproPack["files"] = []
+  if (r.runDir) {
+    for (const [name, what] of [["failure.png", "Last screenshot"], ["workflow.yaml", "Replay flow (forgeqe replay)"], ["steps.json", "Every browser step"], ["network.json", "Every API call, secrets removed"], ["report.html", "The run's report"]]) {
+      const f = path.join(r.runDir, name)
+      if (existsSync(f)) files.push({ name: what, path: rel(f) })
+    }
+  }
+  if (r.code?.knownDefect && r.code.testFile) files.push({ name: "Test that asserts the correct behaviour (an expected failure)", path: r.code.testFile })
+  const request = failingRequest(r.runDir, d)
+  return {
+    ...reproducibility(r, opts.seen, opts.intermittent),
+    steps,
+    actions: browserActions(r.runDir),
+    ...(r.case.startUrl || e?.startUrl ? { startUrl: r.case.startUrl || e?.startUrl } : {}),
+    ...(r.finalUrl ? { endUrl: r.finalUrl } : {}),
+    at: r.finishedAt,
+    env,
+    ...(request ? { request } : {}),
+    files,
+  }
+}
+
+/** The pack as plain text, for a bug's description: reproducibility, the steps, and the failing request. */
+export function reproText(p: ReproPack): string {
+  const lines = [`Reproducibility: ${p.reproducibility} (${p.detail}).`]
+  if (p.startUrl) lines.push(`Start at: ${p.startUrl}`)
+  if (p.actions.length) lines.push("", "What the browser did:", ...p.actions.map((a, i) => `${i + 1}. ${a}`))
+  if (p.request) {
+    lines.push("", `Failing request: ${p.request.method} ${p.request.url}${p.request.status ? ` → ${p.request.status}` : ""}`, "Run it again (secrets already removed):", p.request.curl)
+    if (p.request.responseBody) lines.push(`Response: ${p.request.responseBody}`)
+  }
+  if (p.env.length) lines.push("", "Environment:", ...p.env.map((l) => `- ${l}`))
+  return lines.join("\n")
+}
+
+const fence = (s: string) => `\`\`\`\n${s}\n\`\`\``
+
+/** defects/<id>.md in the suite folder, one per defect with a pack: everything to make it happen again, and the analysis. */
+export function writeReproFiles(suiteDir: string, clusters: DefectCluster[]): void {
+  const dir = path.join(suiteDir, "defects")
+  for (const c of clusters) {
+    const p = c.repro
+    if (!p) continue
+    const rca = c.rca
+    const md = [
+      `# ${c.title}`,
+      "",
+      `${c.failureLabel} · ${c.cases.length} case${c.cases.length === 1 ? "" : "s"} · reproducibility: **${p.reproducibility}** (${p.detail})${c.bug ? ` · bug ${c.bug.url ? `[${c.bug.label}](${c.bug.url})` : c.bug.label}` : ""}`,
+      "",
+      "## Cases that fail this way",
+      ...c.cases.map((x) => `- ${x.id} ${x.title} (${x.status})`),
+      "",
+      "## Evidence",
+      ...c.evidence.map((e) => `- ${e}`),
+      "",
+      "## To reproduce",
+      ...(p.startUrl ? [`Start at ${p.startUrl}.`, ""] : []),
+      ...p.steps.map((s) => `- ${s}`),
+      ...(p.actions.length ? ["", "What the browser did:", ...p.actions.map((a, i) => `${i + 1}. ${a}`)] : []),
+      ...(p.endUrl ? ["", `It ended on ${p.endUrl}.`] : []),
+      ...(p.request
+        ? ["", "## Failing request", `${p.request.method} ${p.request.url}${p.request.status ? ` → ${p.request.status}` : ""}. Secrets were removed when the run recorded it.`, "", fence(p.request.curl), ...(p.request.requestBody ? ["", "Request body:", fence(p.request.requestBody)] : []), ...(p.request.responseBody ? ["", "Response body:", fence(p.request.responseBody)] : [])]
+        : []),
+      "",
+      "## Environment",
+      ...p.env.map((l) => `- ${l}`),
+      ...(p.files.length ? ["", "## Files", ...p.files.map((f) => `- ${f.name}: \`${f.path}\``)] : []),
+      ...(rca ? ["", "## Likely cause (a hypothesis, not a verdict)", ...rcaLines(rca)] : []),
+      "",
+    ].join("\n")
+    mkdirSync(dir, { recursive: true })
+    const name = `${c.id}.md`
+    writeFileSync(path.join(dir, name), md)
+    p.file = `defects/${name}`
+  }
+}
+
+/** The analysis as markdown lines, for the repro file and the bug comment. */
+export function rcaLines(rca: NonNullable<DefectCluster["rca"]>): string[] {
+  return [
+    `- Layer: ${rca.layer}${rca.component ? ` · component: ${rca.component}` : ""} (${rca.basis})`,
+    ...(rca.hypothesis ? [`- Hypothesis (${rca.confidence || "low"} confidence): ${rca.hypothesis}`] : []),
+    ...(rca.evidence?.length ? ["- Based on:", ...rca.evidence.map((e) => `  - ${e}`)] : []),
+    ...(rca.confirm?.length ? ["- To confirm:", ...rca.confirm.map((e) => `  - ${e}`)] : []),
+    ...(rca.files?.length ? [`- Code to look at: ${rca.files.join(", ")}`] : []),
+    ...(rca.suspects?.length ? [`- Suspect commits (${rca.range}):`, ...rca.suspects.map((s) => `  - ${s.sha.slice(0, 10)} ${s.subject} (${s.author}, ${s.at.slice(0, 10)}): ${s.why}`)] : rca.range ? [`- No commit ${rca.range} mentions it.`] : []),
+    ...(rca.note ? [`- ${rca.note}`] : []),
+    ...(rca.model ? [`- Written by ${rca.model}${rca.tokens ? `, ${rca.tokens} tokens` : ""}${rca.cost ? `, ${rca.cost.amount.toFixed(2)} ${rca.cost.currency}` : ""}.`] : []),
+  ]
+}

package/src/cases/types.ts

@@ -1,10 +1,11 @@
 import type { CodeStatus, StepVerdict, Usage } from "../types.ts"
 import type { Cost } from "../usage.ts"
+import type { CaseDefect, Reproduced } from "./defects.ts"
 import type { Failure } from "./failure.ts"
 
 export type CaseStep = { action: string; data?: string; expected?: string }
 
-export type CaseSource = "text" | "file" | "ado" | "jira" | "xray" | "testrail" | "zephyr" | "qtest"
+export type CaseSource = "text" | "file" | "ado" | "jira" | "xray" | "testrail" | "zephyr" | "qtest" | "github"
 
 /** A test case from any source, in one shape. It becomes one browser goal and, once it passes, one test. */
 export type TestCase = {
@@ -33,6 +34,8 @@ export type CaseStatus = "passed" | "failed" | "stopped" | "error"
 export type CaseResult = {
   case: TestCase
   status: CaseStatus
+  /** The named device this result ran on (browser_devices); its case id and title carry the device too. */
+  device?: string
   /** The browser agent's account of the run, or why it could not run. */
   summary: string
   startedAt: string
@@ -73,6 +76,10 @@ export type CaseResult = {
   codeSkipped?: string
   /** For a case that did not pass: what kind of failure it was, and so what happened next (see src/cases/failure.ts). */
   failure?: Failure
+  /** For a case that failed or could not run: the defect it belongs to, shared with every case failing the same way. */
+  defect?: CaseDefect
+  /** The case was run once more to see whether its defect repeats (the first case of each new defect). */
+  reproduced?: Reproduced
   /** The bug filed (or found open) for a functional defect, when bugs are set up. */
   bug?: { label: string; url?: string }
   /** The browser run was repeated once because the first one met an environment problem. */

package/src/cli/help.ts

@@ -14,7 +14,7 @@ Usage:
   forgeqe features [--config forgeqe.yaml]   what ForgeQE can do, and what each capability still needs here
   forgeqe docs [--guide|--open] [--copy DIR]  the guides and the architecture page (--guide: the user guide in the browser; --open: the architecture page)
   forgeqe start [--profile name] [--url URL] [--goal TEXT] [--headed] [--replay MODE] [--no-memory]
-                 [--repo DIR] [--no-code]
+                 [--repo DIR] [--no-code] [--browser-provider local|browserstack|testmu] [--devices a,b]
   forgeqe run "<request>" [--profile name] [--url URL] [--headed] [--plain] [--max-steps N]
                [--repo DIR] [--no-code] [--replay MODE] [--no-memory] [--attempts N] [--code-model provider/model]
                [--browser-provider local|browserstack|testmu]
@@ -23,7 +23,7 @@ Usage:
   forgeqe run --direct "<goal>" [--out DIR] [--skill NAME] [--browser-provider local|browserstack|testmu] ...  one browser goal, no agent
   forgeqe suite <source...> [--parallel N] [--repo DIR] [--no-code] [--no-write-back] [--out DIR]
                  [--profile name] [--url URL] [--headed] [--plain] [--attempts N] [--skill NAME]
-                 [--browser-provider local|browserstack|testmu]
+                 [--browser-provider local|browserstack|testmu] [--devices a,b]
                                                run test cases the same way every time (what pipelines call)
   forgeqe cases <source...> [--goal]          list the test cases a source gives (--goal: the goal each runs)
   forgeqe code [run-dir] [--repo DIR] [--attempts N] [--code-model provider/model]
@@ -49,6 +49,9 @@ Usage:
                                                them and exits 1 when one fails (a CI gate), --post puts the results on the PR
   forgeqe quality [run-dir]                   a run's WCAG rules passed and failed, Core Web Vitals, and budgets
   forgeqe quality trends [page]               accessibility and timings per page over time: what got worse or better
+  forgeqe defects [--folder DIR]              defects over the saved suites: still there, gone, came back, time to fix, by area
+  forgeqe defects predict [--repo DIR] [--base REF] [--head REF]
+                                               where a change is likely to cause defects, with reasons and the cases to run first
   forgeqe notify test [channel]                  send a test card to the Teams and email notification channels
   forgeqe eval domain [--only id,id] [--min 0.8]  does the agent apply domain and regulatory knowledge? (with the domain declared and without)
   forgeqe prompts [list]                      every agent prompt and where it comes from (ForgeQE's or your edited copy)
@@ -112,6 +115,9 @@ page with ranked locators and a confidence %, plus the steps of each goal's last
 --replay trust    replay and finish without the LLM when every step works
 --replay off      never replay; still record pages
 --browser-provider  pick which configured browser provider runs this command's browser work
+--devices a,b       run on each named device (devices: in the BrowserStack/TestMu YAML file; in the shell, describe
+                    them and ForgeQE's agent saves them): devices run side by side, each reported on its own, and each
+                    case posts one combined result back (it fails when any device fails)
 
 ForgeQE's agent (forgeqe start, forgeqe run) runs on the agent runtime with the active profile's model (or code.model) and
 calls ForgeQE's tools: browser runs, reading and running test cases, writing tests and running them, page memory, pushing