@capagents/forgeqe 1.2.0 → 1.3.0

package/src/pack/pack.ts

@@ -0,0 +1,327 @@
+import { existsSync, mkdirSync, readdirSync, readFileSync, realpathSync, writeFileSync } from "node:fs"
+import path from "node:path"
+
+import type { AppConfig } from "../config.ts"
+import { areaFacts, predict } from "../cases/predict.ts"
+import { buildGraph, qualified, testsReaching, type CodeGraph } from "../code/graph/graph.ts"
+import { flakyReport } from "../code/history.ts"
+import { analyzeImpact } from "../code/impact.ts"
+import { walk } from "../code/profile.ts"
+import { indexTests, shortlistTests, testsOnAreas, type TestEntry } from "../code/shortlist.ts"
+import { readMappings } from "../code/traceability.ts"
+import { readPages } from "../memory.ts"
+import { projectDir } from "../tools.ts"
+import type { ChangeItem } from "./collect.ts"
+
+/**
+ * A regression pack: the tests a release's changes can break, each with why it is in, saved under a name so it can be
+ * run again (locally, on TestMu AI, or in a pipeline) and its runs compared. A test is in for one or more reasons:
+ *
+ * - linked: it automates a test case that tests a changed work item (Tested By, through .forgeqe/test-cases.json, or a
+ *   case id in the test's attributes);
+ * - changed: the change is in the test repository, and the test reaches the changed code (imports, class names);
+ * - covers: the work item's or pull request's words name what the test does (the shortlist: page memory elements,
+ *   pages, and test names);
+ * - risk: the change touches an area with defects or failures before (defect prediction), and the test works there;
+ * - smoke: it is tagged smoke, so it goes in every pack.
+ *
+ * Changes no test is in for are the pack's gaps: what nobody tests yet.
+ */
+
+export type ReasonKind = "linked" | "changed" | "covers" | "risk" | "smoke"
+export type PackReason = { kind: ReasonKind; text: string; item?: string }
+export type PackTest = { test: string; file: string; line?: number; reasons: PackReason[]; score: number; flaky?: string }
+export type PackRun = {
+  at: string
+  on: "local" | "testmu" | "pipeline"
+  target?: string
+  status: string
+  counts: { passed: number; failed: number; skipped: number }
+  seconds: number
+  failed: string[]
+  dir?: string
+  url?: string
+}
+export type Pack = {
+  version: 1
+  name: string
+  createdAt: string
+  repo: string
+  scope: string[]
+  items: ChangeItem[]
+  tests: PackTest[]
+  /** Changes no test is in the pack for; with the closest test, a start for extending one rather than writing one. */
+  uncovered: { item: string; title: string; why: string; nearest?: { test: string; file: string; lacks: string[] } }[]
+  risk: { area: string; level: string; reasons: string[] }[]
+  notes: string[]
+  runs: PackRun[]
+}
+
+/** A work item's or PR's words must match a test at least this closely for it to count as covering it. */
+const COVER_FROM = 0.45
+/** A test reaching one element a change names is its closest only this close: one generic element is not enough. */
+const NEAREST_FROM = 0.35
+const COVER_TOP = 3
+const WEIGHT: Record<ReasonKind, number> = { linked: 1, changed: 0.9, covers: 0.8, risk: 0.6, smoke: 0.3 }
+const SMOKE = /(^|[^a-z])smoke([^a-z]|$)/i
+
+const itemName = (i: ChangeItem) => `${i.kind === "pr" ? "PR " : i.kind === "commit" ? "commit " : ""}${i.source === "azure_devops" && i.kind === "work_item" ? `${i.type || "work item"} ` : ""}${i.id}`
+
+export const packsDir = (repo: string) => path.join(projectDir(repo), "packs")
+const packFile = (repo: string, name: string) => path.join(packsDir(repo), `${name}.json`)
+
+export function packName(given?: string, now = new Date()): string {
+  const slug = (given || "").toLowerCase().replace(/[^a-z0-9._-]+/g, "-").replace(/^-+|-+$/g, "")
+  return slug || `release-${now.toISOString().slice(0, 16).replace(/[-:T]/g, "").replace(/^(\d{8})(\d{4})$/, "$1-$2")}`
+}
+
+export function savePack(p: Pack): string {
+  mkdirSync(packsDir(p.repo), { recursive: true })
+  const file = packFile(p.repo, p.name)
+  writeFileSync(file, `${JSON.stringify(p, null, 2)}\n`)
+  return file
+}
+
+export function loadPack(repo: string, name: string): Pack {
+  const file = packFile(repo, name)
+  if (!existsSync(file)) {
+    const known = listPacks(repo).map((p) => p.name)
+    throw new Error(`No pack named "${name}" in ${packsDir(repo)}.${known.length ? ` Packs: ${known.join(", ")}.` : " Build one first."}`)
+  }
+  return JSON.parse(readFileSync(file, "utf8")) as Pack
+}
+
+export function listPacks(repo: string): Pack[] {
+  const dir = packsDir(repo)
+  if (!existsSync(dir)) return []
+  return readdirSync(dir)
+    .filter((f) => f.endsWith(".json"))
+    .flatMap((f) => {
+      try {
+        return [JSON.parse(readFileSync(path.join(dir, f), "utf8")) as Pack]
+      } catch {
+        return []
+      }
+    })
+    .sort((a, b) => b.createdAt.localeCompare(a.createdAt))
+}
+
+/** The pack for these changes: every test a change can break, with why; and the changes no test covers. */
+export async function buildPack(opts: { config: AppConfig; repo: string; name?: string; scope: string[]; items: ChangeItem[]; notes?: string[]; risk?: boolean; smoke?: boolean; graph?: CodeGraph }): Promise<Pack> {
+  const cfg = opts.config
+  let repo = opts.repo
+  try {
+    repo = realpathSync(repo)
+  } catch {}
+  const graph = opts.graph || (await buildGraph(repo))
+  const index = indexTests(graph).map((t) => t.entry)
+  const pages = cfg.memory.enabled ? readPages(cfg.memory.dir) : []
+  const notes = [...(opts.notes || [])]
+  const tests = new Map<string, PackTest>()
+  const reached = new Set<string>()
+  const add = (t: Pick<TestEntry, "test" | "file"> & { line?: number }, reason: PackReason) => {
+    const key = `${t.file}#${t.test}`
+    const e = tests.get(key) || { test: t.test, file: t.file, ...(t.line ? { line: t.line } : {}), reasons: [], score: 0 }
+    if (!e.reasons.some((r) => r.kind === reason.kind && r.text === reason.text && r.item === reason.item)) e.reasons.push(reason)
+    tests.set(key, e)
+    if (reason.item) reached.add(reason.item)
+  }
+  const inFile = (file: string) => index.filter((t) => t.file === file)
+
+  // linked: test cases that test a changed work item, through the mappings or the tests' own case ids.
+  const mappings = readMappings(repo).cases
+  for (const item of opts.items) {
+    for (const caseId of item.testCases || []) {
+      const m = mappings[caseId]
+      const why = { kind: "linked" as const, text: `automates ${caseId}, which tests ${itemName(item)} "${item.title}"`, item: item.id }
+      if (m?.testFile && m.status !== "needs_attention") {
+        const named = m.testName ? index.filter((t) => t.file === m.testFile && (m.testName === t.test || m.testName!.endsWith(`.${t.test}`) || m.testName!.endsWith(t.test.split(".").pop()!))) : []
+        const hits = named.length ? named : inFile(m.testFile)
+        if (hits.length) for (const t of hits) add(t, why)
+        else add({ test: m.testName || m.testFile, file: m.testFile }, why)
+      }
+      for (const t of index.filter((x) => x.tags.some((tag) => tag.toUpperCase() === caseId.toUpperCase()))) add(t, why)
+    }
+  }
+
+  // changed: changes in the test repository reach tests through its code. Per changed file, every change that touched
+  // it gets the credit: tests in the file, tests that call its members, tests whose class inherits one of its classes
+  // (a base test class), and, for JS and Python, tests that import it.
+  const testFiles = [...new Set(opts.items.filter((i) => i.inTestRepo).flatMap((i) => i.files || []))]
+  if (testFiles.length) {
+    const owners = (file: string) => opts.items.filter((i) => i.inTestRepo && (i.files || []).includes(file))
+    const credit = (file: string, t: Pick<TestEntry, "test" | "file"> & { line?: number }, text: string) => {
+      const who = owners(file)
+      if (!who.length) add(t, { kind: "changed", text })
+      for (const i of who) add(t, { kind: "changed", text, item: i.id })
+    }
+    const classesIn = new Map(graph.files.map((f) => [f.file, f.classes.map((c) => c.name)]))
+    const basesOf = new Map(graph.files.flatMap((f) => f.classes.map((c) => [c.name, c.bases] as const)))
+    const inherits = (cls: string | undefined, target: Set<string>) => {
+      const seen = new Set<string>()
+      let todo = cls ? [cls] : []
+      while (todo.length) {
+        const c = todo.pop()!
+        if (seen.has(c)) continue
+        seen.add(c)
+        for (const b of basesOf.get(c) || []) {
+          if (target.has(b)) return b
+          todo.push(b)
+        }
+      }
+      return undefined
+    }
+    const byKey = new Map(index.map((t) => [`${t.file}#${t.test}`, t]))
+    for (const file of testFiles.slice(0, 200)) {
+      for (const t of inFile(file)) credit(file, t, `is in ${file}, which changed`)
+      for (const r of testsReaching(graph, [file])) {
+        const e = byKey.get(`${r.test.file}#${qualified(r.test)}`)
+        if (e && e.file !== file) credit(file, e, `${r.why.replace(/^calls /, "calls ")} (${file} changed)`)
+      }
+      const changedClasses = new Set(classesIn.get(file) || [])
+      if (changedClasses.size) {
+        for (const t of index) {
+          const cls = t.test.includes(".") ? t.test.slice(0, t.test.lastIndexOf(".")) : undefined
+          const base = inherits(cls, changedClasses)
+          if (base) credit(file, t, `inherits ${base} (${file} changed)`)
+        }
+      }
+    }
+    // Imports, for JS and Python; a file named like a test but holding none is not one.
+    const impact = analyzeImpact(repo, testFiles.filter((f) => /\.([cm]?[jt]sx?|py)$/.test(f)))
+    for (const t of impact.tests) {
+      const hits = inFile(t.file)
+      const text = t.reasons.join("; ")
+      const file = testFiles.find((f) => text.includes(f)) || t.file
+      if (hits.length) for (const e of hits) credit(file, e, text)
+    }
+  }
+
+  // covers: the item's words match what a test does. The closest one is kept for a change no test covers.
+  const nearest = new Map<string, { test: string; file: string; lacks: string[] }>()
+  for (const item of opts.items) {
+    if (item.inTestRepo && item.kind === "commit") continue
+    const request = `${item.title}\n${item.text || ""}`.slice(0, 1500)
+    const s = shortlistTests({ graph, pages, request, max: COVER_TOP })
+    // Closest only when it reaches an element the change names: a shared word alone ("checkout") is not a start.
+    const close = s.candidates.find((c) => c.covers.length >= 2 || (c.covers.length === 1 && c.score >= NEAREST_FROM))
+    if (close) nearest.set(item.id, { test: close.test, file: close.file, lacks: close.lacks })
+    for (const c of s.candidates.filter((c) => c.score >= COVER_FROM)) {
+      add(c, { kind: "covers", text: `matches ${itemName(item)} "${item.title}"${c.why.length ? ` (${c.why.join("; ")})` : ""}`, item: item.id })
+    }
+  }
+
+  // risk: changed product code in areas with defects or failures before, and the tests that work there.
+  const risk: Pack["risk"] = []
+  if (opts.risk !== false) {
+    const product = [...new Set(opts.items.filter((i) => !i.inTestRepo).flatMap((i) => i.files || []))]
+    if (product.length) {
+      const facts = areaFacts(cfg.suite.outDir)
+      const p = predict(facts, { files: product, diff: new Map() })
+      const areas = p.areas.filter((a) => a.level !== "low")
+      for (const a of areas) risk.push({ area: a.area, level: a.level, reasons: a.reasons })
+      const on = testsOnAreas(graph, pages, areas.map((a) => a.area))
+      for (const t of index) {
+        const hit = on.get(`${t.file}#${t.test}`)
+        if (hit) add(t, { kind: "risk", text: `works on ${hit.join(", ")}, a ${areas.find((a) => hit.includes(a.area))?.level} risk area this release changes` })
+      }
+      for (const caseId of p.runFirst) {
+        const m = mappings[caseId]
+        if (m?.testFile) for (const t of inFile(m.testFile)) add(t, { kind: "risk", text: `${caseId} is one to run first for the risk areas` })
+      }
+    }
+  }
+
+  // smoke: in every pack.
+  if (opts.smoke !== false) for (const t of index.filter((x) => x.tags.some((tag) => SMOKE.test(tag)) || SMOKE.test(x.test))) add(t, { kind: "smoke", text: "tagged smoke: in every pack" })
+
+  // Flaky tests are marked, so their failures are read as flaky first.
+  const flaky = flakyReport(repo).filter((f) => f.flaky)
+  for (const t of tests.values()) {
+    const f = flaky.find((x) => x.name === t.test || x.name.endsWith(`.${t.test}`) || x.name.endsWith(t.test))
+    if (f) t.flaky = `flaky: ${f.flips ?? "several"} flips in its recent runs`
+    const kinds = [...new Set(t.reasons.map((r) => r.kind))]
+    t.score = Number((Math.max(...kinds.map((k) => WEIGHT[k])) + 0.1 * (kinds.length - 1)).toFixed(2))
+  }
+  const uncovered = opts.items
+    .filter((i) => !reached.has(i.id))
+    .map((i) => ({
+      item: i.id,
+      title: i.title,
+      why: [i.testCases?.length ? `its test cases ${i.testCases.join(", ")} have no automated test here` : "no test case is linked", i.files?.length && i.inTestRepo ? "no test reaches its files" : "", "no test matches what it describes"].filter(Boolean).join("; "),
+      ...(nearest.has(i.id) ? { nearest: nearest.get(i.id)! } : {}),
+    }))
+  if (!index.length) notes.push(`ForgeQE found no tests in ${repo}: the pack can only list the changes.`)
+  const pack: Pack = {
+    version: 1,
+    name: packName(opts.name),
+    createdAt: new Date().toISOString(),
+    repo,
+    scope: opts.scope,
+    items: opts.items,
+    tests: [...tests.values()].sort((a, b) => b.score - a.score || a.file.localeCompare(b.file) || a.test.localeCompare(b.test)),
+    uncovered,
+    risk,
+    notes,
+    runs: [],
+  }
+  savePack(pack)
+  return pack
+}
+
+const quote = (s: string) => (/^[\w./@:+-]+$/.test(s) ? s : `'${s.replace(/'/g, `'\\''`)}'`)
+
+/** Past this the dotnet filter names the classes instead of each test: a command line has a limit. */
+const MAX_FILTER = 6000
+
+/**
+ * The command that runs exactly the pack's tests, writing results into `resultsDir`, built the way the repository
+ * already runs a test (an earlier job's command: its configuration and settings) or the framework's usual form.
+ */
+export function packCommand(repo: string, tests: Pick<PackTest, "test" | "file">[], resultsDir: string, knownCommands: string[] = [], files = walk(repo)): { command: string; framework: string; notes: string[] } | null {
+  const notes: string[] = []
+  const parts: { command: string; framework: string }[] = []
+  const cs = tests.filter((t) => t.file.endsWith(".cs"))
+  if (cs.length) {
+    const projects = files.filter((f) => f.endsWith(".csproj"))
+    const projectOf = (file: string) => projects.filter((p) => file.startsWith(`${path.posix.dirname(p)}/`)).sort((a, b) => b.length - a.length)[0]
+    const known = knownCommands.find((c) => /\bdotnet\s+test\b/.test(c) && /--filter/.test(c))
+    const byProject = new Map<string, typeof cs>()
+    for (const t of cs) {
+      const p = projectOf(t.file)
+      if (!p) {
+        notes.push(`${t.file} is in no test project: left out of the command.`)
+        continue
+      }
+      byProject.set(p, [...(byProject.get(p) || []), t])
+    }
+    for (const [project, list] of byProject) {
+      let filter = list.map((t) => `FullyQualifiedName~${t.test}`).join("|")
+      if (filter.length > MAX_FILTER) {
+        filter = [...new Set(list.map((t) => t.test.split(".").slice(0, -1).join(".") || t.test))].map((c) => `FullyQualifiedName~${c}`).join("|")
+        notes.push(`The pack has too many tests in ${project} for one filter: its classes run whole.`)
+      }
+      const logger = `--logger "trx;LogFileName=${path.posix.basename(project, ".csproj")}.trx" --results-directory ${quote(resultsDir)}`
+      // An earlier job's command for this project: its configuration and settings, its filter swapped; never headed.
+      const base = known && known.includes(project) ? known.replace(/^(?:\s*HEADED=\S+\s+)+/, "").replace(/--filter\s+("[^"]*"|\S+)/, `--filter "${filter}"`) : `dotnet test ${project} --filter "${filter}"`
+      parts.push({ framework: "dotnet", command: `${base} ${logger}` })
+    }
+  }
+  const js = tests.filter((t) => /\.[cm]?[jt]sx?$/.test(t.file))
+  if (js.length) {
+    if (files.some((f) => /(^|\/)playwright\.config\.(ts|js|mjs|cjs)$/.test(f))) {
+      const titles = [...new Set(js.map((t) => t.test.split(".").pop()!.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")))]
+      parts.push({ framework: "Playwright", command: `PLAYWRIGHT_JUNIT_OUTPUT_NAME=${quote(path.join(resultsDir, "playwright.xml"))} npx playwright test ${[...new Set(js.map((t) => quote(t.file)))].join(" ")} -g ${quote(titles.join("|"))} --reporter=line,junit` })
+    } else notes.push(`${js.length} JavaScript tests are not Playwright Test tests: left out of the command.`)
+  }
+  const py = tests.filter((t) => t.file.endsWith(".py"))
+  if (py.length) {
+    const ids = py.map((t) => (t.test === t.file ? t.file : `${t.file}::${t.test.replace(/\./g, "::")}`))
+    parts.push({ framework: "pytest", command: `python -m pytest ${ids.map(quote).join(" ")} --junitxml=${quote(path.join(resultsDir, "pytest.xml"))}` })
+  }
+  const other = tests.length - cs.length - js.length - py.length
+  if (other) notes.push(`${other} tests in other languages are left out of the command.`)
+  if (!parts.length) return null
+  // Every part runs, whatever the one before it did; the results files say what passed.
+  return { command: parts.map((p) => p.command).join(" ; "), framework: [...new Set(parts.map((p) => p.framework))].join(" + "), notes }
+}