@capagents/forgeqe 1.1.4 → 1.1.6

package/src/cases/bugs.ts

@@ -6,6 +6,9 @@ import type { BugsConfig, SourcesConfig } from "../config.ts"
 import { browserErrors, problemLine } from "../report/data.ts"
 import { ciInfo } from "../report/html.ts"
 import { adoSettings } from "./ado.ts"
+import { DefectBook, type CaseDefect, type DefectCluster } from "./defects.ts"
+import { rcaLines, reproPack, reproText } from "./repro.ts"
+import { githubBugTracker } from "./github.ts"
 import { http, textToAdf } from "./http.ts"
 import { jiraApi, type JiraApi } from "./jira.ts"
 import { stepVerdictLines, type CaseResult, type WriteBack } from "./types.ts"
@@ -57,35 +60,82 @@ export function bugText(r: CaseResult, runName: string): string {
 const failing = (r: CaseResult, b: BugsConfig) =>
   r.failure ? r.failure.type === "functional_defect" : r.status === "failed" || (b.onError && r.status === "error")
 
-/** File a bug for each failing case, or comment on the open one filed before; note on the open bug when a case passes again. */
-export async function fileBugs(results: CaseResult[], cfg: SourcesConfig, runName: string): Promise<WriteBack[]> {
+/** A defect from browser evidence holds across runs and cases; a "page" defect (the agent's words) only within a run. */
+const lasting = (d?: CaseDefect): d is CaseDefect => !!d && d.kind !== "page"
+
+/** The bug's title: the defect, when cases share it by evidence; else the case. */
+export function defectBugTitle(r: CaseResult, d?: CaseDefect): string {
+  return lasting(d) ? `[ForgeQE] ${d.title}`.slice(0, 250) : bugTitle(r)
+}
+
+/**
+ * The bug's description: the defect's evidence first when it has some, then the case that found it, then how to make
+ * it happen again (src/cases/repro.ts) when the defect is known.
+ */
+export function defectBugText(r: CaseResult, runName: string, d?: CaseDefect, book?: DefectBook): string {
+  const repro = d ? ["", reproText(reproPack(r, d, { env: book?.env, seen: book?.past[d.id]?.seen ?? book?.earlier?.(r.case.id) }))] : []
+  if (!lasting(d)) return [bugText(r, runName), ...repro].join("\n")
+  return [`Defect: ${d.title}`, ...d.evidence.map((e) => `- ${e}`), "", "Every test case that fails this way is added to this bug as a comment.", "", "First found by:", bugText(r, runName), ...repro].join("\n")
+}
+
+/**
+ * The root-cause analysis of each defect added to its bug as a comment, once the suite has it. Only bugs filed or
+ * found in this run, and only analyses with a hypothesis.
+ */
+export async function commentAnalyses(cfg: SourcesConfig, book: DefectBook, clusters: DefectCluster[]): Promise<WriteBack[]> {
+  const b = cfg.bugs
+  const todo = clusters.filter((c) => c.rca?.hypothesis && book.bugs.has(c.id))
+  if (!b || !todo.length) return []
+  let tracker: Tracker
+  try {
+    tracker = b.system === "jira" ? jiraTracker(cfg, b) : b.system === "github" ? githubBugTracker(cfg, b) : adoTracker(cfg, b)
+  } catch (err) {
+    return [{ source: "bugs", text: "Analyses not added to bugs", error: (err as Error).message }]
+  }
+  const out: WriteBack[] = []
+  for (const c of todo) {
+    const bug = book.bugs.get(c.id)!
+    try {
+      await tracker.comment(bug.id, ["Root-cause analysis by ForgeQE: a hypothesis to check, not a verdict.", "", ...rcaLines(c.rca!)].join("\n"))
+      out.push({ source: "bugs", caseId: bug.caseId, text: `${bug.label}: root-cause analysis added`, url: bug.url })
+    } catch (err) {
+      out.push({ source: "bugs", caseId: bug.caseId, text: `${bug.label}: analysis not added`, error: (err as Error).message })
+    }
+  }
+  return out
+}
+
+/**
+ * File a bug for each defect (cases failing the same way share one: src/cases/defects.ts), or for each failing case
+ * when bugs.group_by is case. A case joining a defect's bug is added as a comment and tagged with its own fingerprint,
+ * so the note when it passes again finds that bug. Notes on the open bug when a case passes again.
+ */
+export async function fileBugs(results: CaseResult[], cfg: SourcesConfig, runName: string, book = new DefectBook()): Promise<WriteBack[]> {
   const b = cfg.bugs
   if (!b) return []
   const relevant = results.filter((r) => failing(r, b) || r.status === "passed")
   if (!relevant.length) return []
   let tracker: Tracker
   try {
-    tracker = b.system === "jira" ? jiraTracker(cfg, b) : adoTracker(cfg, b)
+    tracker = b.system === "jira" ? jiraTracker(cfg, b) : b.system === "github" ? githubBugTracker(cfg, b) : adoTracker(cfg, b)
   } catch (err) {
     return [{ source: "bugs", text: "Bugs not filed", error: (err as Error).message }]
   }
   const out: WriteBack[] = []
   for (const r of relevant) {
     const fp = bugFingerprint(r)
+    // A bug filed before ForgeQE was renamed from WebPilot carries the same hash under the old prefix.
+    const byCase = async () => (await tracker.findOpen(fp)) ?? (await tracker.findOpen(fp.replace(/^forgeqe-/, "webpilot-")))
     try {
-      // A bug filed before ForgeQE was renamed from WebPilot carries the same hash under the old prefix.
-      const open = (await tracker.findOpen(fp)) ?? (await tracker.findOpen(fp.replace(/^forgeqe-/, "webpilot-")))
       if (r.status === "passed") {
+        const open = await byCase()
         if (!open) continue
         await tracker.comment(open.id, `ForgeQE: ${r.case.id} passed again in "${runName}".`)
         out.push({ source: "bugs", caseId: r.case.id, text: `${open.label}: noted that ${r.case.id} passes again`, url: open.url })
-      } else if (open) {
-        await tracker.comment(open.id, `Still failing.\n\n${bugText(r, runName)}`)
-        out.push({ source: "bugs", caseId: r.case.id, text: `${open.label}: still failing, comment added`, url: open.url })
-      } else {
-        const bug = await tracker.create(r, fp, runName)
-        out.push({ source: "bugs", caseId: r.case.id, text: `${bug.label}: bug filed for ${r.case.id}${bug.note ? ` (${bug.note})` : ""}`, url: bug.url })
+        continue
       }
+      const d = b.groupBy === "case" ? undefined : (r.defect ?? book.assign(r))
+      out.push(await book.serial(() => fileOne(tracker, b, r, fp, d, runName, book, byCase)))
     } catch (err) {
       out.push({ source: "bugs", caseId: r.case.id, text: `${r.case.id}: bug not filed`, error: (err as Error).message })
     }
@@ -93,12 +143,91 @@ export async function fileBugs(results: CaseResult[], cfg: SourcesConfig, runNam
   return out
 }
 
-type BugRef = { id: string; label: string; url: string; note?: string }
+async function fileOne(tracker: Tracker, b: BugsConfig, r: CaseResult, fp: string, d: CaseDefect | undefined, runName: string, book: DefectBook, byCase: () => Promise<BugRef | null>): Promise<WriteBack> {
+  const id = r.case.id
+  // Another case of this defect filed or found its bug earlier in the run: this case joins it.
+  const shared = d && book.bugs.get(d.id)
+  if (shared) {
+    await tracker.comment(shared.id, `Also fails: ${id}.\n\n${bugText(r, runName)}`)
+    const tagged = await tag(tracker, shared.id, fp)
+    return { source: "bugs", caseId: id, text: `${shared.label}: same defect as ${shared.caseId}, ${id} added${tagged}`, url: shared.url }
+  }
+  const byDefect = lasting(d) ? await tracker.findOpen(d.id) : null
+  const open = byDefect ?? (await byCase())
+  const gone = d && book.past[d.id]?.goneSince
+  if (open) {
+    const head = gone ? `Failing again after it went away on ${gone.slice(0, 10)}` : "Still failing"
+    await tracker.comment(open.id, `${head}${byDefect ? `: ${id}` : ""}.\n\n${bugText(r, runName)}`)
+    // Found by one fingerprint, it gets the other, so the next run finds it either way.
+    const tagged = await tag(tracker, open.id, byDefect ? fp : lasting(d) ? d.id : "")
+    if (d) book.bugs.set(d.id, { ...open, caseId: id })
+    return { source: "bugs", caseId: id, text: `${open.label}: still failing, comment added${tagged}`, url: open.url }
+  }
+  // No open bug: a closed one with either fingerprint means it was fixed and came back.
+  const closed = (lasting(d) ? await tracker.findClosed(d.id) : null) ?? (await tracker.findClosed(fp))
+  const labels = [...new Set([...b.labels, ...(lasting(d) ? [d.id] : []), fp])]
+  if (closed) {
+    const when = closed.closedAt ? ` on ${closed.closedAt.slice(0, 10)}` : ""
+    const body = [`Regression: this came back. ${closed.label} (${closed.url}) was closed${when}.`, "", defectBugText(r, runName, d, book)].join("\n")
+    const bug = await tracker.create(r, { title: `[ForgeQE] Regression: ${defectBugTitle(r, d).replace(/^\[ForgeQE\] /, "")}`.slice(0, 250), body, labels, regressionOf: closed }, runName)
+    if (d) {
+      book.bugs.set(d.id, { ...bug, caseId: id })
+      book.closed.set(d.id, { label: closed.label, url: closed.url, ...(closed.closedAt ? { closedAt: closed.closedAt } : {}) })
+    }
+    return { source: "bugs", caseId: id, text: `${bug.label}: regression of ${closed.label}, bug filed for ${id}${bug.note ? ` (${bug.note})` : ""}`, url: bug.url }
+  }
+  // Bugs people filed by hand about the same route or error: named on the new bug, never merged.
+  const related = lasting(d) && d.term ? await relatedBugs(tracker, d.term) : []
+  const body = defectBugText(r, runName, d, book) + (related.length ? `\n\nPossibly related open bugs (they mention ${d!.term}):\n${related.map((x) => `- ${x.label}: ${x.title} (${x.url})`).join("\n")}` : "")
+  const bug = await tracker.create(r, { title: defectBugTitle(r, d), body, labels }, runName)
+  if (d) {
+    book.bugs.set(d.id, { ...bug, caseId: id })
+    if (related.length) book.related.set(d.id, related.map((x) => ({ label: x.label, url: x.url, title: x.title })))
+  }
+  const also = related.length ? `, ${related.length} possibly related` : ""
+  return { source: "bugs", caseId: id, text: `${bug.label}: bug filed for ${id}${bug.note || also ? ` (${[bug.note, also.slice(2)].filter(Boolean).join(", ")})` : ""}`, url: bug.url }
+}
+
+/** The search for related bugs is a hint: a tracker that cannot run it files the bug without one. */
+async function relatedBugs(tracker: Tracker, term: string): Promise<FoundBug[]> {
+  try {
+    return (await tracker.related(term, 3)).filter((x) => !/^\[(ForgeQE|WebPilot)\]/.test(x.title))
+  } catch {
+    return []
+  }
+}
+
+/** Add a label or tag; a tracker that refuses only loses the note, not the comment. */
+async function tag(tracker: Tracker, id: string, label: string): Promise<string> {
+  if (!label) return ""
+  try {
+    await tracker.tag(id, label)
+    return ""
+  } catch {
+    return ` (not tagged ${label})`
+  }
+}
 
-type Tracker = {
+export type BugRef = { id: string; label: string; url: string; note?: string }
+
+/** A closed bug, and when it was closed when the tracker says. */
+export type ClosedBug = BugRef & { closedAt?: string }
+
+/** An open bug found by its text, with its title. */
+export type FoundBug = BugRef & { title: string }
+
+/** The bug's title, description, and labels (or tags): its fingerprints among them. `regressionOf`: the closed bug it came back from, linked. */
+export type NewBug = { title: string; body: string; labels: string[]; regressionOf?: ClosedBug }
+
+export type Tracker = {
   findOpen(fingerprint: string): Promise<BugRef | null>
-  create(r: CaseResult, fingerprint: string, runName: string): Promise<BugRef>
+  /** The most recently closed bug with this label or tag. */
+  findClosed(fingerprint: string): Promise<ClosedBug | null>
+  /** Open bugs whose text mentions `term`, newest first. */
+  related(term: string, limit: number): Promise<FoundBug[]>
+  create(r: CaseResult, bug: NewBug, runName: string): Promise<BugRef>
   comment(id: string, text: string): Promise<void>
+  tag(id: string, label: string): Promise<void>
 }
 
 function screenshot(r: CaseResult): { name: string; data: Buffer } | null {
@@ -120,16 +249,29 @@ function jiraTracker(cfg: SourcesConfig, b: BugsConfig): Tracker {
       const hit = data?.issues?.[0]
       return hit ? ref(hit.key) : null
     },
-    async create(r, fp, runName) {
+    async findClosed(fp) {
+      const jql = `project = "${b.project}" AND labels = "${fp}" AND statusCategory = Done ORDER BY updated DESC`
+      const q = new URLSearchParams({ jql, fields: "summary,resolutiondate", maxResults: "1" })
+      const { data } = await http<{ issues?: { key: string; fields?: { resolutiondate?: string | null } }[] }>(`${j.api}/${j.cloud ? "search/jql" : "search"}?${q}`, { auth: j.auth })
+      const hit = data?.issues?.[0]
+      return hit ? { ...ref(hit.key), ...(hit.fields?.resolutiondate ? { closedAt: hit.fields.resolutiondate } : {}) } : null
+    },
+    async related(term, limit) {
+      const jql = `project = "${b.project}" AND statusCategory != Done AND text ~ "\\"${term.replace(/["\\]/g, " ")}\\"" ORDER BY updated DESC`
+      const q = new URLSearchParams({ jql, fields: "summary", maxResults: String(limit) })
+      const { data } = await http<{ issues?: { key: string; fields?: { summary?: string } }[] }>(`${j.api}/${j.cloud ? "search/jql" : "search"}?${q}`, { auth: j.auth })
+      return (data?.issues || []).map((i) => ({ ...ref(i.key), title: i.fields?.summary || "" }))
+    },
+    async create(r, bug) {
       const { data } = await http<{ key: string; id: string }>(`${j.api}/issue`, {
         auth: j.auth,
         body: {
           fields: {
             project: { key: b.project },
             issuetype: { name: b.type },
-            summary: bugTitle(r),
-            description: body(bugText(r, runName)),
-            labels: [...new Set([...b.labels, fp])],
+            summary: bug.title,
+            description: body(bug.body),
+            labels: bug.labels,
           },
         },
       })
@@ -145,6 +287,14 @@ function jiraTracker(cfg: SourcesConfig, b: BugsConfig): Tracker {
         })
         notes.push(res.ok ? "screenshot attached" : `screenshot not attached: ${res.status}`)
       }
+      if (bug.regressionOf) {
+        try {
+          await http(`${j.api}/issueLink`, { auth: j.auth, body: { type: { name: "Relates" }, inwardIssue: { key: data.key }, outwardIssue: { key: bug.regressionOf.id } } })
+          notes.push(`linked to ${bug.regressionOf.label}`)
+        } catch {
+          notes.push(`not linked to ${bug.regressionOf.label}`)
+        }
+      }
       const linked = r.case.source === "jira" || r.case.source === "xray" ? String(r.case.ref?.issue || r.case.ref?.test || r.case.id) : ""
       if (linked) {
         try {
@@ -159,6 +309,9 @@ function jiraTracker(cfg: SourcesConfig, b: BugsConfig): Tracker {
     async comment(id, text) {
       await http(`${j.api}/issue/${encodeURIComponent(id)}/comment`, { auth: j.auth, body: { body: body(text) } })
     },
+    async tag(id, label) {
+      await http(`${j.api}/issue/${encodeURIComponent(id)}`, { method: "PUT", auth: j.auth, body: { update: { labels: [{ add: label }] } } })
+    },
   }
 }
 
@@ -181,12 +334,35 @@ function adoTracker(cfg: SourcesConfig, b: BugsConfig): Tracker {
       const hit = data?.workItems?.[0]
       return hit ? ref(hit.id) : null
     },
-    async create(r, fp, runName) {
+    async findClosed(fp) {
+      const query =
+        `SELECT [System.Id] FROM WorkItems WHERE [System.TeamProject] = @project AND [System.WorkItemType] = '${b.type.replace(/'/g, "''")}' ` +
+        `AND [System.Tags] CONTAINS '${fp}' AND [System.State] IN ('Closed', 'Done', 'Resolved') ORDER BY [System.ChangedDate] DESC`
+      const { data } = await http<{ workItems?: { id: number }[] }>(`${base}/_apis/wit/wiql?${api}&$top=1`, { auth, body: { query } })
+      const hit = data?.workItems?.[0]
+      if (!hit) return null
+      // Items created closed (or moved there by a rule) have no ClosedDate: when their state last changed is when they closed.
+      const dates = ["Microsoft.VSTS.Common.ClosedDate", "Microsoft.VSTS.Common.ResolvedDate", "Microsoft.VSTS.Common.StateChangeDate"]
+      const { data: item } = await http<{ fields?: Record<string, string> }>(`${base}/_apis/wit/workitems/${hit.id}?fields=${dates.join(",")}&${api}`, { auth })
+      const at = dates.map((f) => item?.fields?.[f]).find(Boolean)
+      return { ...ref(hit.id), ...(at ? { closedAt: at } : {}) }
+    },
+    async related(term, limit) {
+      const query =
+        `SELECT [System.Id] FROM WorkItems WHERE [System.TeamProject] = @project AND [System.WorkItemType] = '${b.type.replace(/'/g, "''")}' ` +
+        `AND [System.State] NOT IN ('Closed', 'Done', 'Removed', 'Resolved') AND ([System.Title] CONTAINS '${term.replace(/'/g, "''")}' OR [Microsoft.VSTS.TCM.ReproSteps] CONTAINS WORDS '${term.replace(/'/g, "''")}') ORDER BY [System.ChangedDate] DESC`
+      const { data } = await http<{ workItems?: { id: number }[] }>(`${base}/_apis/wit/wiql?${api}&$top=${limit}`, { auth, body: { query } })
+      const ids = (data?.workItems || []).slice(0, limit).map((w) => w.id)
+      if (!ids.length) return []
+      const { data: items } = await http<{ value?: { id: number; fields?: Record<string, string> }[] }>(`${base}/_apis/wit/workitems?ids=${ids.join(",")}&fields=System.Title&${api}`, { auth })
+      return (items?.value || []).map((w) => ({ ...ref(w.id), title: w.fields?.["System.Title"] || "" }))
+    },
+    async create(r, bug) {
       const notes: string[] = []
       const ops: Record<string, unknown>[] = [
-        { op: "add", path: "/fields/System.Title", value: bugTitle(r) },
-        { op: "add", path: "/fields/Microsoft.VSTS.TCM.ReproSteps", value: html(bugText(r, runName)) },
-        { op: "add", path: "/fields/System.Tags", value: [...new Set([...b.labels, fp])].join("; ") },
+        { op: "add", path: "/fields/System.Title", value: bug.title },
+        { op: "add", path: "/fields/Microsoft.VSTS.TCM.ReproSteps", value: html(bug.body) },
+        { op: "add", path: "/fields/System.Tags", value: bug.labels.join("; ") },
       ]
       if (b.areaPath) ops.push({ op: "add", path: "/fields/System.AreaPath", value: b.areaPath })
       const testCase = r.case.source === "ado" ? Number(r.case.ref?.workItem) : 0
@@ -194,6 +370,10 @@ function adoTracker(cfg: SourcesConfig, b: BugsConfig): Tracker {
         ops.push({ op: "add", path: "/relations/-", value: { rel: "Microsoft.VSTS.Common.TestedBy-Forward", url: `${ado.orgUrl}/_apis/wit/workItems/${testCase}` } })
         notes.push(`linked to test case ${testCase}`)
       }
+      if (bug.regressionOf) {
+        ops.push({ op: "add", path: "/relations/-", value: { rel: "System.LinkTypes.Related", url: `${ado.orgUrl}/_apis/wit/workItems/${bug.regressionOf.id}`, attributes: { comment: "Regression of" } } })
+        notes.push(`linked to ${bug.regressionOf.label}`)
+      }
       const shot = screenshot(r)
       if (shot) {
         const res = await fetch(`${base}/_apis/wit/attachments?fileName=${encodeURIComponent(shot.name)}&${api}`, {
@@ -219,5 +399,12 @@ function adoTracker(cfg: SourcesConfig, b: BugsConfig): Tracker {
     async comment(id, text) {
       await http(`${base}/_apis/wit/workItems/${id}/comments?api-version=7.1-preview.4`, { auth, body: { text: html(text) } })
     },
+    async tag(id, label) {
+      // Setting System.Tags replaces them all: read the ones it has first.
+      const { data } = await http<{ fields?: Record<string, string> }>(`${base}/_apis/wit/workitems/${id}?fields=System.Tags&${api}`, { auth })
+      const tags = (data?.fields?.["System.Tags"] || "").split(";").map((t) => t.trim()).filter(Boolean)
+      if (tags.includes(label)) return
+      await http(`${base}/_apis/wit/workitems/${id}?${api}`, { method: "PATCH", auth, contentType: "application/json-patch+json", body: [{ op: "add", path: "/fields/System.Tags", value: [...tags, label].join("; ") }] })
+    },
   }
 }

package/src/cases/defectHistory.ts

@@ -0,0 +1,193 @@
+import { existsSync, readdirSync, readFileSync, writeFileSync } from "node:fs"
+import path from "node:path"
+
+import type { DefectCluster } from "./defects.ts"
+import type { CaseResult, CaseStatus } from "./types.ts"
+
+/**
+ * Defect intelligence, part two: what earlier runs know about each defect. Kept in defect-history.json next to the
+ * suite folders (the folder the report reads earlier runs from; keep it between CI runs as a cache to track a
+ * pipeline's defects). Only defects found by browser evidence are kept: a defect matched by the agent's words gets
+ * new words, and so a new id, every run.
+ *
+ * - new: never seen before
+ * - known: seen in earlier runs, and still there
+ * - regression: it went away (a case it broke passed, and it was not seen) and came back; or its bug was closed
+ */
+
+export const DEFECT_HISTORY = "defect-history.json"
+const KEEP_SIGHTINGS = 30
+const KEEP_CASES = 50
+/** Earlier suites read to tell an intermittent failure from a steady one. */
+const WINDOW = 20
+
+export type Sighting = { at: string; suite: string; cases: string[] }
+
+export type PastDefect = {
+  title: string
+  kind: string
+  /** The page it was seen on last, for the area it belongs to. */
+  page?: string
+  firstSeen: string
+  lastSeen: string
+  /** Runs it was seen in. */
+  seen: number
+  sightings: Sighting[]
+  /** The cases it has broken, newest last. */
+  cases: string[]
+  bug?: { label: string; url?: string }
+  /** It went away: not seen in a run where a case it broke passed. */
+  goneSince?: string
+  /** When it came back after going away. */
+  returns?: string[]
+}
+
+export type DefectHistoryFile = { version: 1; defects: Record<string, PastDefect> }
+
+export type DefectHistoryNote = {
+  status: "new" | "known" | "regression"
+  /** Earlier runs it was seen in. */
+  seen: number
+  firstSeen?: string
+  /** For a regression: since when it was gone, or the closed bug it matches. */
+  goneSince?: string
+  previousBug?: { label: string; url?: string; closedAt?: string }
+  /** Its cases kept changing between passing and failing: failed in `failed` of the last `runs` runs. */
+  intermittent?: { failed: number; runs: number }
+}
+
+export const defectHistoryFile = (suiteDir: string) => path.join(path.dirname(suiteDir), DEFECT_HISTORY)
+
+export function readDefectHistory(file: string): DefectHistoryFile {
+  try {
+    const d = JSON.parse(readFileSync(file, "utf8")) as DefectHistoryFile
+    return { version: 1, defects: d?.defects && typeof d.defects === "object" ? d.defects : {} }
+  } catch {
+    return { version: 1, defects: {} }
+  }
+}
+
+export function writeDefectHistory(file: string, h: DefectHistoryFile): void {
+  writeFileSync(file, `${JSON.stringify(h, null, 2)}\n`)
+}
+
+type SavedSuite = { at: string; results: { case: { id: string }; status: CaseStatus }[] }
+
+/** The suites saved next to this one (this one left out), oldest first. */
+function savedSuites(suiteDir: string): SavedSuite[] {
+  const parent = path.dirname(suiteDir)
+  const self = path.basename(suiteDir)
+  const suites: SavedSuite[] = []
+  let names: string[] = []
+  try {
+    names = readdirSync(parent).filter((n) => n !== self && existsSync(path.join(parent, n, "suite.json")))
+  } catch {}
+  for (const n of names) {
+    try {
+      const s = JSON.parse(readFileSync(path.join(parent, n, "suite.json"), "utf8"))
+      if (s?.startedAt && Array.isArray(s.results)) suites.push({ at: s.startedAt, results: s.results })
+    } catch {}
+  }
+  return suites.sort((a, b) => a.at.localeCompare(b.at))
+}
+
+/** Each case's results in the suites saved next to this one, oldest first (this suite left out). */
+export function caseOutcomes(suiteDir: string, window = WINDOW): Map<string, CaseStatus[]> {
+  const out = new Map<string, CaseStatus[]>()
+  for (const s of savedSuites(suiteDir).slice(-window)) {
+    for (const r of s.results) {
+      if (r.status === "stopped") continue
+      const list = out.get(r.case.id) || []
+      list.push(r.status)
+      out.set(r.case.id, list)
+    }
+  }
+  return out
+}
+
+/** When the newest saved suite in which one of these cases passed started; undefined when none did. */
+export function lastPassed(suiteDir: string, caseIds: string[]): string | undefined {
+  const ids = new Set(caseIds)
+  return [...savedSuites(suiteDir)].reverse().find((s) => s.results.some((r) => ids.has(r.case.id) && r.status === "passed"))?.at
+}
+
+/** How many of the latest runs in a row failed: 0 when the latest passed or there is none. */
+export function failedInARow(statuses: CaseStatus[] = []): number {
+  let n = 0
+  for (let i = statuses.length - 1; i >= 0 && statuses[i] !== "passed"; i--) n++
+  return n
+}
+
+/** Failed in some runs and passed in others, changing more than once: pass, fail, pass. */
+function intermittent(statuses: CaseStatus[]): { failed: number; runs: number } | undefined {
+  const bad = statuses.map((s) => s !== "passed")
+  const flips = bad.slice(1).filter((b, i) => b !== bad[i]).length
+  return flips >= 2 ? { failed: bad.filter(Boolean).length, runs: bad.length } : undefined
+}
+
+/**
+ * What is known about each of this run's defects: from the history, the closed bug a regression matched in the tracker,
+ * and the earlier results of its cases (with this run's result counted).
+ */
+export function noteHistory(clusters: DefectCluster[], h: DefectHistoryFile, outcomes: Map<string, CaseStatus[]>, closed: Map<string, { label: string; url?: string; closedAt?: string }> = new Map()): void {
+  for (const c of clusters) {
+    const past = c.kind === "page" ? undefined : h.defects[c.id]
+    const previousBug = closed.get(c.id)
+    // A defect matched by the agent's words has no history of its own: its cases failing in the runs just before
+    // make it known, so it is not run again and analysed again every suite.
+    const streak = c.kind === "page" ? Math.min(...c.cases.map((x) => failedInARow(outcomes.get(x.id)))) : 0
+    const status = past?.goneSince || previousBug ? "regression" : past || streak > 0 ? "known" : "new"
+    // A regression's pass in between was the fix, not chance.
+    const flaky = status === "regression" ? undefined : c.cases.map((x) => intermittent([...(outcomes.get(x.id) || []), x.status])).find(Boolean)
+    c.history = {
+      status,
+      seen: past?.seen || streak,
+      ...(past ? { firstSeen: past.firstSeen } : {}),
+      ...(past?.goneSince ? { goneSince: past.goneSince } : {}),
+      ...(previousBug || (past?.goneSince && past.bug) ? { previousBug: previousBug || past!.bug } : {}),
+      ...(flaky ? { intermittent: flaky } : {}),
+    }
+  }
+}
+
+/** Add this run: its defects are seen now; an earlier defect not seen while a case it broke passed has gone away. */
+export function recordRun(h: DefectHistoryFile, clusters: DefectCluster[], results: CaseResult[], suite: string, at: string): void {
+  const seen = new Set<string>()
+  for (const c of clusters) {
+    if (c.kind === "page") continue
+    seen.add(c.id)
+    const cases = c.cases.map((x) => x.id)
+    const p: PastDefect = h.defects[c.id] || { title: c.title, kind: c.kind, firstSeen: at, lastSeen: at, seen: 0, sightings: [], cases: [] }
+    if (p.goneSince) {
+      p.returns = [...(p.returns || []), at].slice(-KEEP_SIGHTINGS)
+      delete p.goneSince
+    }
+    p.title = c.title
+    if (c.page || c.pages[0]) p.page = c.page || c.pages[0]
+    p.lastSeen = at
+    p.seen += 1
+    p.sightings = [...p.sightings, { at, suite, cases }].slice(-KEEP_SIGHTINGS)
+    p.cases = [...p.cases.filter((id) => !cases.includes(id)), ...cases].slice(-KEEP_CASES)
+    if (c.bug) p.bug = c.bug
+    h.defects[c.id] = p
+  }
+  const passed = new Set(results.filter((r) => r.status === "passed").map((r) => r.case.id))
+  for (const [id, p] of Object.entries(h.defects)) {
+    if (!seen.has(id) && !p.goneSince && p.cases.some((c) => passed.has(c))) p.goneSince = at
+  }
+}
+
+/** "new", "seen in 3 earlier runs since 2026-10-02", "regression of SHOP-12 (closed 2026-10-01)", "intermittent: …". */
+export function historyText(n?: DefectHistoryNote): string {
+  if (!n) return ""
+  const day = (at?: string) => (at ? at.slice(0, 10) : "")
+  const base =
+    n.status === "regression"
+      ? n.previousBug
+        ? `regression of ${n.previousBug.label}${n.previousBug.closedAt ? ` (closed ${day(n.previousBug.closedAt)})` : n.goneSince ? ` (gone since ${day(n.goneSince)})` : ""}`
+        : `regression (gone since ${day(n.goneSince)})`
+      : n.status === "known"
+        ? `seen in ${n.seen} earlier run${n.seen === 1 ? "" : "s"}${n.firstSeen ? ` since ${day(n.firstSeen)}` : ""}`
+        : "new"
+  return n.intermittent ? `${base}; intermittent: failed ${n.intermittent.failed} of the last ${n.intermittent.runs} runs` : base
+}