@capagents/forgeqe 1.1.4 → 1.1.6

package/src/run.ts

@@ -24,6 +24,8 @@ export type ExecuteOptions = {
   config: AppConfig
   goal: string
   browserProvider?: BrowserProvider
+  /** A named device from the provider files (browser_devices); it also picks the provider. */
+  browserDevice?: string
   url?: string
   headed?: boolean
   maxSteps?: number
@@ -149,14 +151,14 @@ export async function executeRun(opts: ExecuteOptions): Promise<{ result: RunRes
 
   const dir = opts.outDir || runDirectory(opts.config.export.outDir)
   const maxSteps = opts.maxSteps || opts.config.browser.maxSteps
-  const browser = browserConfigFor(opts.config, opts.browserProvider)
+  const browser = browserConfigFor(opts.config, opts.browserProvider, opts.browserDevice)
   const t0 = Date.now()
   const withReport = (out: { result: RunResult; files: string[] }) => {
     out.result.cost = costOf(out.result.usage, opts.config.llm.price)
     applyQuality(opts.config, dir, out.result, opts.onEvent)
     logEvent("run.done", { out_dir: dir, status: out.result.status, summary: out.result.summary, usage: out.result.usage, memory: out.result.memory, ...(out.result.skills ? { skills: out.result.skills } : {}) })
     try {
-      out.files.push(writeRunReport(dir, out.result, reportEnv(opts.config, { url: startUrl, headed: opts.headed }), t0))
+      out.files.push(writeRunReport(dir, out.result, reportEnv(opts.config, { url: startUrl, headed: opts.headed, browser }), t0))
     } catch (err) {
       opts.onEvent?.({ type: "log", level: "warn", text: `report.html not written: ${(err as Error).message}` })
     }

package/src/suite/report.ts

@@ -1,6 +1,8 @@
 import { writeFileSync } from "node:fs"
 import path from "node:path"
 
+import { defectLine } from "../cases/defects.ts"
+import { historyText } from "../cases/defectHistory.ts"
 import { stepVerdictLines, type CaseResult } from "../cases/types.ts"
 import { renderReport } from "../report/html.ts"
 import type { Usage } from "../types.ts"
@@ -25,8 +27,9 @@ export function counts(results: CaseResult[]) {
 export function writeReports(s: Omit<SuiteResult, "files">): SuiteResult["files"] {
   const files = { junit: path.join(s.dir, "junit.xml"), html: path.join(s.dir, "report.html"), json: path.join(s.dir, "suite.json") }
   writeFileSync(files.junit, junit(s))
-  writeFileSync(files.html, renderReport({ title: s.name, startedAt: s.startedAt, ms: s.ms, stopped: s.stopped, dir: s.dir, results: s.results, posted: s.writeBack, env: s.env }))
+  // suite.json first: the report's trends read the saved suites, this one included.
   writeFileSync(files.json, `${JSON.stringify({ ...s, files }, null, 2)}\n`)
+  writeFileSync(files.html, renderReport({ title: s.name, startedAt: s.startedAt, ms: s.ms, stopped: s.stopped, dir: s.dir, results: s.results, posted: s.writeBack, env: s.env, defects: s.defects }))
   return files
 }
 
@@ -60,17 +63,16 @@ function junit(s: Omit<SuiteResult, "files">): string {
 }
 
 /** Every case's browser and Scribe tokens together, and their cost (unset when any part has no price). */
-export function suiteUsage(results: CaseResult[]): { usage: Usage; cost?: Cost } {
+export function suiteUsage(results: CaseResult[], analysis?: { usage: Usage; cost?: Cost }): { usage: Usage; cost?: Cost } {
   let usage: Usage = { input: 0, output: 0 }
   let cost: Cost | undefined
   let priced = true
-  for (const r of results) {
-    for (const [u, c] of [[r.usage, r.cost], [r.codeUsage, r.codeCost]] as const) {
-      if (!u || (!u.input && !u.output)) continue
-      usage = addUsage(usage, u)
-      if (c) cost = addCost(cost, c)
-      else priced = false
-    }
+  const parts = [...results.flatMap((r) => [[r.usage, r.cost], [r.codeUsage, r.codeCost]] as const), ...(analysis ? [[analysis.usage, analysis.cost] as const] : [])]
+  for (const [u, c] of parts) {
+    if (!u || (!u.input && !u.output)) continue
+    usage = addUsage(usage, u)
+    if (c) cost = addCost(cost, c)
+    else priced = false
   }
   return { usage, cost: priced ? cost : undefined }
 }
@@ -80,9 +82,11 @@ export function formatSuite(s: SuiteResult, cwd = process.cwd()): string {
   const c = counts(s.results)
   const tokens = s.results.reduce((n, r) => n + r.tokens, 0)
   const head = `${s.stopped ? "■ Suite stopped" : c.passed === c.total ? "✓ Suite passed" : "✗ Suite failed"} · ${c.passed}/${c.total} passed · ${duration(s.ms)} · ${(tokens / 1000).toFixed(1)}K tokens`
-  const total = suiteUsage(s.results)
+  const show = (f: string) => path.relative(cwd, f) || f
+  const total = suiteUsage(s.results, s.analysis)
   const lines = [head, ""]
   if (total.usage.input || total.usage.output) lines.push(`Usage     ${usageLine(total.usage, total.cost)}`, "")
+  if (s.analysis) lines.push(`Analysis  ${s.analysis.defects} root-cause ${s.analysis.defects === 1 ? "analysis" : "analyses"}: ${usageLine(s.analysis.usage, s.analysis.cost)} (in the usage above)`, "")
   for (const r of s.results) {
     const code = r.code ? `  test ${r.code.status}${r.code.testFile ? `: ${r.code.testFile}` : ""}` : ""
     lines.push(`${MARK[r.status]} ${r.case.id}  ${r.case.title}  (${duration(r.ms)})${code}`)
@@ -90,11 +94,20 @@ export function formatSuite(s: SuiteResult, cwd = process.cwd()): string {
     const failedStep = r.stepResults?.find((v) => v.status === "failed")
     if (r.status !== "passed" && failedStep) lines.push(`    step ${failedStep.step} failed${failedStep.actual ? `: ${firstLine(failedStep.actual)}` : ""}`)
   }
+  const defects = s.defects || []
+  if (worthListing(defects)) {
+    lines.push("", `Defects   ${defectLine(defects)}`)
+    for (const d of defects) {
+      lines.push(`  ${String(d.cases.length).padStart(2)}× ${d.title}${d.history ? `  (${historyText(d.history)})` : ""}${d.bug ? `  ${d.bug.label}` : ""}`)
+      if (d.rca?.hypothesis) lines.push(`      likely (${d.rca.confidence}): ${firstLine(d.rca.hypothesis)}`)
+      else if (d.rca) lines.push(`      likely layer: ${d.rca.layer}${d.rca.suspects?.length ? `; ${d.rca.suspects.length} suspect commit${d.rca.suspects.length === 1 ? "" : "s"}` : ""}`)
+      if (d.repro?.file) lines.push(`      repro: ${show(path.join(s.dir, d.repro.file))} (${d.repro.reproducibility})`)
+    }
+  }
   if (s.writeBack.length) {
     lines.push("")
     for (const w of s.writeBack) lines.push(`${w.error ? "✗" : "↑"} ${w.text}${w.url ? `  ${w.url}` : ""}${w.error ? `: ${w.error}` : ""}`)
   }
-  const show = (f: string) => path.relative(cwd, f) || f
   lines.push("", `Report    ${show(s.files.html)}`, `JUnit     ${show(s.files.junit)}`)
   return lines.join("\n")
 }
@@ -112,17 +125,37 @@ export function markdownSummary(s: SuiteResult): string {
     return `| ${MARK[r.status]} | ${cell(r.case.id)} | ${cell(r.case.title)} | ${steps} | ${duration(r.ms)} | ${result} | ${test} |`
   })
   const posted = s.writeBack.filter((w) => !w.caseId || w.error).map((w) => `- ${w.error ? "✗" : "↑"} ${w.text}${w.url ? ` ${w.url}` : ""}${w.error ? `: ${w.error}` : ""}`)
+  const defects = s.defects || []
+  const defectRows = worthListing(defects)
+    ? [
+        "",
+        `#### Defects: ${defectLine(defects)}`,
+        "",
+        "| Cases | Defect | Kind | History | Reproducible | Likely cause | Bug |",
+        "|---|---|---|---|---|---|---|",
+        ...defects.map(
+          (d) =>
+            `| ${d.cases.length} | ${cell(d.title)} | ${cell(d.failureLabel)} | ${cell(historyText(d.history))} | ${d.repro?.reproducibility || ""} | ${cell(d.rca ? (d.rca.hypothesis ? `${d.rca.hypothesis} (${d.rca.confidence})` : d.rca.layer) : "")} | ${d.bug ? (d.bug.url ? `[${cell(d.bug.label)}](${d.bug.url})` : cell(d.bug.label)) : ""} |`,
+        ),
+      ]
+    : []
   return [
     `### ${head} · ${c.passed}/${c.total} passed · ${duration(s.ms)}`,
     "",
     "| | Case | Title | Steps | Time | Result | Test |",
     "|---|---|---|---|---|---|---|",
     ...rows,
+    ...defectRows,
     ...(posted.length ? ["", ...posted] : []),
     "",
   ].join("\n")
 }
 
+/** Defects are listed when they add to the table of cases: cases share one, or one has been seen before. */
+function worthListing(defects: NonNullable<SuiteResult["defects"]>): boolean {
+  return defects.some((d) => d.cases.length > 1 || (d.history && d.history.status !== "new") || d.history?.intermittent)
+}
+
 function firstLine(s: string): string {
   return s.split("\n")[0].slice(0, 200)
 }

package/src/suite/run.ts

@@ -2,12 +2,18 @@ import { existsSync, mkdirSync, writeFileSync } from "node:fs"
 import path from "node:path"
 
 import { goalFor, writeBack as postResults, type CaseResult, type TestCase, type WriteBack } from "../cases/index.ts"
-import { costOf } from "../cases/types.ts"
+import { costOf, type Cost } from "../cases/types.ts"
 import { addUsage, NO_USAGE } from "../usage.ts"
-import type { AgentEvent, CodeResult, ReplayMode, RunResult } from "../types.ts"
+import type { AgentEvent, CodeResult, ReplayMode, RunResult, Usage } from "../types.ts"
 import { codeSkipReason, runCodeAgent } from "../code/index.ts"
 import type { AppConfig, BrowserProvider } from "../config.ts"
 import { fileBugs } from "../cases/bugs.ts"
+import { DefectBook, defectClusters, type DefectCluster } from "../cases/defects.ts"
+import { analyzeCause, basicCause } from "../cases/rca.ts"
+import { scorePrediction } from "../cases/predict.ts"
+import { reproPack, writeReproFiles } from "../cases/repro.ts"
+import { commentAnalyses } from "../cases/bugs.ts"
+import { caseOutcomes, defectHistoryFile, failedInARow, noteHistory, readDefectHistory, recordRun, writeDefectHistory } from "../cases/defectHistory.ts"
 import { classifyFailure, FAILURE_LABEL } from "../cases/failure.ts"
 import { associateAdoAutomation, readMappings } from "../code/traceability.ts"
 import type { ExistingTest } from "../code/prompt.ts"
@@ -36,6 +42,10 @@ export type SuiteResult = {
   files: { junit: string; html: string; json: string }
   stopped: boolean
   env?: ReportEnv
+  /** Failures grouped by cause: one entry per defect with every case it breaks (src/cases/defects.ts). */
+  defects?: DefectCluster[]
+  /** The model's root-cause analyses in this suite: how many, and what they used (src/cases/rca.ts). */
+  analysis?: { defects: number; usage: Usage; cost?: Cost }
 }
 
 export type SuiteOptions = {
@@ -44,6 +54,8 @@ export type SuiteOptions = {
   /** Repo for the coding agent; unset: browser runs only. */
   repo?: string
   browserProvider?: BrowserProvider
+  /** Named devices: every case runs once on each, and each case posts one combined result back to its source. */
+  devices?: string[]
   parallel?: number
   outDir?: string
   writeBack?: boolean
@@ -66,6 +78,21 @@ export type SuiteOptions = {
 
 const STATUS = { complete: "passed", fail: "failed", stopped: "stopped" } as const
 
+/** A case as run on one device: the device in its id and title keeps reports, history, and the shell apart. */
+export function deviceCase(tc: TestCase, device: string): TestCase {
+  return { ...tc, id: `${tc.id}@${device}`, title: `${tc.title} · ${device}` }
+}
+
+/** The cases a suite shows: each case once per device, in the order they run. */
+export function deviceCases(cases: TestCase[], devices: string[]): TestCase[] {
+  return devices.length ? cases.flatMap((tc) => devices.map((d) => deviceCase(tc, d))) : cases
+}
+
+/** Browsers at once: devices run side by side (up to 8) unless more is asked; remote grids queue past their plan's limit. */
+export function suiteParallel(asked: number | undefined, configured: number, devices: number, jobs: number): number {
+  return Math.max(1, Math.min(asked || Math.max(configured, Math.min(devices, 8)), jobs))
+}
+
 function slug(s: string): string {
   return s.replace(/[^A-Za-z0-9._-]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 60) || "case"
 }
@@ -86,33 +113,53 @@ export async function runSuite(opts: SuiteOptions): Promise<SuiteResult> {
   const t0 = Date.now()
   const startedAt = new Date(t0).toISOString()
   const name = `ForgeQE suite ${startedAt.slice(0, 16).replace("T", " ")}`
-  const total = cases.length
+  // One job per case, or per case and device: a device's result carries the device in its case id and title, so
+  // reports, history, and the shell keep each device apart.
+  const devices = [...new Set(opts.devices || [])]
+  const jobs = devices.length
+    ? cases.flatMap((tc) => devices.map((device) => ({ tc, device, shown: deviceCase(tc, device) })))
+    : cases.map((tc) => ({ tc, device: undefined as string | undefined, shown: tc }))
+  const multi = devices.length > 1
+  const total = jobs.length
   const results: (CaseResult | undefined)[] = Array.from({ length: total }, () => undefined)
   const width = String(total).length
   let codeQueue: Promise<void> = Promise.resolve()
   /** Written back during the suite: bugs for functional defects (the suite's own bug pass skips those cases) and tests set as a test case's automation. */
   const earlyBugs: WriteBack[] = []
+  /** Which failures share a cause, and the bug each defect has in this run. */
+  const book = new DefectBook()
+  const historyFile = defectHistoryFile(dir)
+  const history = readDefectHistory(historyFile)
+  book.past = history.defects
+  const env = reportEnv(config, { url: opts.url, headed: opts.headed })
+  book.env = env
+  const earlier = caseOutcomes(dir)
+  const failedBefore = (id: string) => failedInARow(earlier.get(id))
+  book.earlier = failedBefore
+  // A config built in code without defects: (tests, other tools) neither reruns cases nor calls a model for them.
+  const defectsCfg = config.defects ?? { reproduce: false, analyze: false, maxAnalyses: 0 }
 
   const runOne = async (index: number) => {
-    const tc = cases[index]
+    const { tc: original, device, shown: tc } = jobs[index]
     const runDir = path.join(dir, `${String(index + 1).padStart(width, "0")}-${slug(tc.id)}`)
     mkdirSync(runDir, { recursive: true })
     writeFileSync(path.join(runDir, "testcase.json"), `${JSON.stringify(tc, null, 2)}\n`)
     emit({ type: "case-start", index, total, case: tc, runDir })
     const start = Date.now()
-    const explore = async (): Promise<{ run?: RunResult; error?: string }> => {
+    const explore = async (outDir = runDir): Promise<{ run?: RunResult; error?: string }> => {
       try {
         const { result } = await execute({
           config,
-          goal: goalFor(tc),
+          goal: goalFor(original),
           browserProvider: opts.browserProvider,
+          browserDevice: device,
           url: tc.startUrl || opts.url,
           headed: opts.headed,
           maxSteps: opts.maxSteps,
           replay: opts.replay,
           memory: opts.memory,
           caseSteps: tc.steps.length,
-          outDir: runDir,
+          outDir,
           skills: opts.skills,
           repo: opts.skillsRepo || opts.repo,
           signal: opts.signal,
@@ -138,6 +185,7 @@ export async function runSuite(opts: SuiteOptions): Promise<SuiteResult> {
     const end = Date.now()
     const result: CaseResult = {
       case: tc,
+      ...(device ? { device } : {}),
       // Steps that passed on a page over an accessibility or performance budget: the case fails on the budget.
       status: run ? (run.status === "complete" && run.budgetExceeded?.length ? "failed" : STATUS[run.status]) : opts.signal?.aborted ? "stopped" : "error",
       summary: [run?.summary || error || "", ...(run?.budgetExceeded?.length ? [`Budget exceeded: ${run.budgetExceeded.join("; ")}`] : [])].filter(Boolean).join("\n\n"),
@@ -153,19 +201,45 @@ export async function runSuite(opts: SuiteOptions): Promise<SuiteResult> {
       ...(failure ? { failure } : {}),
       ...(retried ? { retried } : {}),
     }
+    if (result.status === "passed") book.notePassed(result)
+    const defect = book.assign(result)
+    if (defect) result.defect = defect
+    // The first case of a new product defect runs once more: a defect that repeats is one a developer can reproduce.
+    // A known defect needs no second run (its history shows it repeating), nor does a problem of the environment.
+    // A case that failed in the suite before has shown it repeats already.
+    if (defect && defectsCfg.reproduce && failure?.type === "functional_defect" && !book.past[defect.id] && !failedBefore(tc.id) && !book.reproduced.has(defect.id) && !opts.signal?.aborted) {
+      book.reproduced.add(defect.id)
+      const again = path.join(runDir, "reproduce")
+      mkdirSync(again, { recursive: true })
+      emit({ type: "case-event", index, case: tc, event: { type: "log", level: "info", text: `New defect (${defect.title}): running ${tc.id} once more to see whether it repeats.` } })
+      const second = await explore(again)
+      if (second.run?.usage) {
+        spent = addUsage(spent || NO_USAGE, second.run.usage)
+        result.usage = spent
+        result.tokens = spent.input + spent.output
+        result.cost = costOf(spent, config.llm.price)
+      }
+      const status: CaseResult["status"] = second.run ? STATUS[second.run.status] : "error"
+      const twin: CaseResult = { ...result, status, summary: second.run?.summary || second.error || "", runDir: again, finalUrl: second.run?.finalUrl || undefined, failure: classifyFailure(second.run, second.error) }
+      const same = book.peek(twin)?.id === defect.id
+      result.reproduced = { runs: 2, failed: same ? 2 : 1, runDir: again }
+      emit({ type: "case-event", index, case: tc, event: { type: "log", level: same ? "info" : "warn", text: same ? "It failed the same way again: reproducible." : `The second run ${status === "passed" ? "passed" : "failed differently"}: the defect did not repeat.` } })
+    }
     results[index] = result
     emit({ type: "case-done", index, total, result })
 
-    // A functional defect's bug is filed now, so the test written for it can name the bug it waits on.
-    if (failure?.type === "functional_defect" && (opts.writeBack ?? config.sources.writeBack) && config.sources.bugs) {
-      for (const item of await fileBugs([result], config.sources, name)) {
+    // A functional defect's bug is filed now, so the test written for it can name the bug it waits on. On several
+    // devices the bug waits for the combined result, so a defect seen on two devices is one bug.
+    if (!multi && failure?.type === "functional_defect" && (opts.writeBack ?? config.sources.writeBack) && config.sources.bugs) {
+      for (const item of await fileBugs([result], config.sources, name, book)) {
         earlyBugs.push(item)
         emit({ type: "writeback", item })
         if (!item.error) result.bug = { label: item.text.split(":")[0], ...(item.url ? { url: item.url } : {}) }
       }
     }
 
-    if (!opts.repo || !config.suite.code || opts.signal?.aborted) return
+    // One test per case: written from its run on the first device.
+    if (!opts.repo || !config.suite.code || opts.signal?.aborted || (device && device !== devices[0])) return
     const skip = run ? codeSkipReason(config, run, failure) : "the browser run did not finish"
     if (skip) {
       result.codeSkipped = skip
@@ -173,7 +247,7 @@ export async function runSuite(opts: SuiteOptions): Promise<SuiteResult> {
     }
     // This case goes from its browser run straight to the coding agent, and its worker waits for that before taking
     // the next case. Coding jobs still queue one after another, since they all edit the same repository.
-    const job = codeQueue.then(() => codeOne(index, tc, runDir, result))
+    const job = codeQueue.then(() => codeOne(index, original, runDir, result))
     codeQueue = job.catch(() => {})
     await job
   }
@@ -263,20 +337,68 @@ export async function runSuite(opts: SuiteOptions): Promise<SuiteResult> {
   const finished = results.map((r, i): CaseResult => {
     if (r) return r
     const now = new Date().toISOString()
-    return { case: cases[i], status: "stopped", summary: "Not run: the suite was stopped.", startedAt: now, finishedAt: now, ms: 0, tokens: 0 }
+    return { case: jobs[i].shown, ...(jobs[i].device ? { device: jobs[i].device } : {}), status: "stopped", summary: "Not run: the suite was stopped.", startedAt: now, finishedAt: now, ms: 0, tokens: 0 }
   })
 
   const posted: WriteBack[] = []
   if (opts.writeBack ?? config.sources.writeBack) {
-    const ran = finished.filter((r) => r.runDir)
+    const ran = devices.length ? combineDevices(cases, jobs, finished).filter((r) => r.runDir) : finished.filter((r) => r.runDir)
     const filed = new Set(earlyBugs.filter((b) => b.source === "bugs").map((b) => b.caseId))
     posted.push(...earlyBugs)
-    for (const item of [...(await postResults(ran, config.sources, name)), ...(await fileBugs(ran.filter((r) => !filed.has(r.case.id)), config.sources, name))]) {
+    for (const item of [...(await postResults(ran, config.sources, name)), ...(await fileBugs(ran.filter((r) => !filed.has(r.case.id)), config.sources, name, book))]) {
+      posted.push(item)
+      emit({ type: "writeback", item })
+    }
+  }
+
+  // What earlier runs and the tracker know about each defect, then this run added to the history.
+  const defects = defectClusters(finished, posted)
+  for (const d of defects) {
+    const related = book.related.get(d.id)
+    if (related) d.related = related
+  }
+  noteHistory(defects, history, earlier, book.closed)
+
+  // How to make each defect happen again, and why it may happen: the layer and suspect commits for every defect, a
+  // model's hypothesis for the new ones and the regressions (the product's defects only, a few per suite).
+  const resultOf = (id: string) => finished.find((r) => r.case.id === id)
+  const productRepo = defectsCfg.productRepo
+  let analyses = 0
+  let analysisUsage: Usage = { ...NO_USAGE }
+  for (const d of defects) {
+    const first = resultOf(d.cases[0].id)
+    if (!first) continue
+    d.repro = reproPack(first, d, { env, seen: d.history?.status === "known" ? d.history.seen : 0, intermittent: d.history?.intermittent, suiteDir: dir })
+    d.rca = basicCause(d, { suiteDir: dir, productRepo })
+    const worth = d.failure === "functional_defect" && (d.history?.status === "new" || d.history?.status === "regression")
+    if (!defectsCfg.analyze || !worth || analyses >= defectsCfg.maxAnalyses || opts.signal?.aborted) continue
+    analyses++
+    const index = finished.indexOf(first)
+    emit({ type: "case-event", index, case: first.case, event: { type: "log", level: "info", text: `Root-cause analysis of ${d.title}${productRepo ? ` (reading ${productRepo})` : " (evidence only: set defects.product_repo to let it read the code)"}` } })
+    d.rca = await analyzeCause(config, d, first, d.rca, { cwd: opts.repo || process.cwd(), productRepo, signal: opts.signal, onEvent: (event) => emit({ type: "case-event", index, case: first.case, event }) })
+    if (d.rca.usage) analysisUsage = addUsage(analysisUsage, d.rca.usage)
+  }
+  try {
+    writeReproFiles(dir, defects)
+  } catch {}
+  if ((opts.writeBack ?? config.sources.writeBack) && analyses) {
+    for (const item of await commentAnalyses(config.sources, book, defects)) {
       posted.push(item)
       emit({ type: "writeback", item })
     }
   }
 
+  if (!opts.signal?.aborted) {
+    // The newest defect prediction made before this suite: did its defects fall where it said?
+    try {
+      scorePrediction(path.dirname(dir), path.basename(dir), startedAt, defects)
+    } catch {}
+    recordRun(history, defects, finished, path.basename(dir), startedAt)
+    try {
+      writeDefectHistory(historyFile, history)
+    } catch {}
+  }
+
   const finishedAt = new Date().toISOString()
   const suite: Omit<SuiteResult, "files"> = {
     name,
@@ -287,11 +409,36 @@ export async function runSuite(opts: SuiteOptions): Promise<SuiteResult> {
     results: finished,
     writeBack: posted,
     stopped: Boolean(opts.signal?.aborted),
-    env: reportEnv(config, { url: opts.url, headed: opts.headed }),
+    env,
+    defects,
+    ...(analyses ? { analysis: { defects: analyses, usage: analysisUsage, ...(costOf(analysisUsage, config.llm.price) ? { cost: costOf(analysisUsage, config.llm.price) } : {}) } } : {}),
   }
   return { ...suite, files: writeReports(suite) }
 }
 
+const WORST: CaseResult["status"][] = ["failed", "error", "stopped", "passed"]
+
+/**
+ * One result per case from its device results, for posting back: the worst status wins (a pass on one device never
+ * hides a failure on another), the summary lists every device, and the worst run's evidence (folder, failure, steps,
+ * bug) goes with it.
+ */
+export function combineDevices(cases: TestCase[], jobs: { tc: TestCase; device?: string }[], results: CaseResult[]): CaseResult[] {
+  return cases.map((tc) => {
+    const own = results.filter((_, i) => jobs[i].tc === tc)
+    const worst = [...own].sort((a, b) => WORST.indexOf(a.status) - WORST.indexOf(b.status))[0]
+    const lines = own.map((r) => `${r.device}: ${r.status}${r.summary ? `: ${(r.summary.split("\n")[0] || "").slice(0, 200)}` : ""}`)
+    return {
+      ...worst,
+      case: tc,
+      device: undefined,
+      summary: [`Ran on ${own.length} device${own.length === 1 ? "" : "s"}:`, ...lines.map((l) => `- ${l}`)].join("\n"),
+      ms: own.reduce((n, r) => n + r.ms, 0),
+      tokens: own.reduce((n, r) => n + r.tokens, 0),
+    }
+  })
+}
+
 function timelineLines(runDir: string): CaseResult["steps"] | undefined {
   const steps = readTimeline(runDir).flatMap((t) => t.actions.filter((a) => !a.done).map((a) => ({ text: a.error ? `${a.text}: ${a.error}` : a.text, ok: a.ok })))
   return steps.length ? steps : undefined