@capagents/forgeqe 1.1.3 → 1.1.5

package/src/cases/readiness.ts

@@ -68,6 +68,38 @@ export function verdictOf(r: Pick<Readiness, "stories" | "bugs" | "counts" | "pa
   return { verdict: blockers.length ? "NO-GO" : risks.length ? "GO WITH RISKS" : "GO", blockers, risks }
 }
 
+type SavedDefect = { title: string; failureLabel?: string; cases: { id: string }[]; bug?: { label: string }; history?: { status?: string; previousBug?: { label: string } } }
+
+/**
+ * What the newest suite with defects says about the test cases in scope (Azure DevOps ids): each regression is a
+ * blocker, each new defect without a bug a risk. Cases are matched by their work item (ADO-123, or the id in ref).
+ */
+export function defectSignals(suitesDir: string, inScope: Set<number>): { blockers: string[]; risks: string[] } {
+  const out = { blockers: [] as string[], risks: [] as string[] }
+  if (!existsSync(suitesDir) || !inScope.size) return out
+  const dirs = readdirSync(suitesDir).filter((d) => existsSync(path.join(suitesDir, d, "suite.json"))).sort().reverse()
+  for (const dir of dirs) {
+    let suite: { results?: { case: { id: string; ref?: { workItem?: number } } }[]; defects?: SavedDefect[] }
+    try {
+      suite = JSON.parse(readFileSync(path.join(suitesDir, dir, "suite.json"), "utf8"))
+    } catch {
+      continue
+    }
+    if (!Array.isArray(suite.defects)) continue
+    const workItem = new Map((suite.results || []).map((r) => [r.case.id, Number(r.case.ref?.workItem || /^ADO-(\d+)/.exec(r.case.id)?.[1] || 0)]))
+    const ids = (d: SavedDefect) => d.cases.map((c) => workItem.get(c.id) || workItem.get(c.id.split("@")[0]) || Number(/^ADO-(\d+)/.exec(c.id)?.[1] || 0)).filter((n) => inScope.has(n))
+    for (const d of suite.defects) {
+      const hit = ids(d)
+      if (!hit.length) continue
+      const cases = `test case${hit.length === 1 ? "" : "s"} ${[...new Set(hit)].join(", ")}`
+      if (d.history?.status === "regression") out.blockers.push(`regression: ${d.title} came back${d.history.previousBug ? ` after ${d.history.previousBug.label}` : ""} (${cases})${d.bug ? `, ${d.bug.label}` : ""}`)
+      else if (d.history?.status === "new" && !d.bug) out.risks.push(`new defect with no bug: ${d.title} (${cases})`)
+    }
+    break
+  }
+  return out
+}
+
 /** The newest result per test case in ForgeQE's suite reports. */
 function forgeqeResults(suitesDir: string): Map<number, { outcome: Outcome; when: string }> {
   const out = new Map<number, { outcome: Outcome; when: string }>()
@@ -228,7 +260,12 @@ export async function releaseReadiness(cfg: AppConfig, repo: string, scope: Scop
   }
   const passRate = executed.length ? Math.round((passed / executed.length) * 1000) / 10 : null
   const v = verdictOf({ stories, bugs, counts, passRate }, thresholds)
-  return { scope: label, stories, bugs, counts, passRate, ...v, notes }
+  // ForgeQE's latest suite: a regression on a case in scope blocks the release; a new defect without a bug is a risk.
+  const inScope = new Set(stories.flatMap((st) => st.cases.map((c) => c.id)))
+  const d = defectSignals(cfg.suite.outDir, inScope)
+  const blockers = [...v.blockers, ...d.blockers]
+  const risks = [...v.risks, ...d.risks]
+  return { scope: label, stories, bugs, counts, passRate, verdict: blockers.length ? "NO-GO" : risks.length ? "GO WITH RISKS" : "GO", blockers, risks, notes }
 }
 
 const MARK: Record<Outcome, string> = { passed: "✅ passed", failed: "❌ failed", blocked: "⛔ blocked", "not run": "⚪ not run" }

package/src/cases/repro.ts

@@ -0,0 +1,202 @@
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"
+import path from "node:path"
+
+import type { ReportEnv } from "../report/html.ts"
+import { readTimeline } from "../report/data.ts"
+import { routeOf, type CaseDefect, type DefectCluster } from "./defects.ts"
+import { stepVerdictLines, type CaseResult } from "./types.ts"
+
+/**
+ * Defect intelligence, part three (a): how to make a defect happen again. Built from what the run of its first case
+ * recorded, with nothing new sent anywhere: the case's steps with the agent's verdicts, what the browser did, where
+ * and with what it ran, and the failing request as a curl command. The engine has already taken credentials out of
+ * the requests it records (py/forgeqe_engine/network.py), so the command carries no secret.
+ */
+
+export type Reproducibility = "always" | "intermittent" | "once"
+
+export type FailingRequest = { method: string; url: string; status?: number; requestBody?: string; responseBody?: string; curl: string }
+
+export type ReproPack = {
+  reproducibility: Reproducibility
+  /** Why: "failed the same way in 2 of 2 runs", "seen in 3 earlier runs". */
+  detail: string
+  /** The case's steps, with the agent's verdict on each when it gave one. */
+  steps: string[]
+  /** What the browser did before it failed: the actions that worked, in order. */
+  actions: string[]
+  startUrl?: string
+  endUrl?: string
+  /** When the run ended (ISO): the report shows it in the reader's time; the env line gives it in UTC. */
+  at: string
+  env: string[]
+  request?: FailingRequest
+  /** Files from the run that help: its folder relative to the suite folder, and each file. */
+  files: { name: string; path: string }[]
+  /** The repro file written for the defect, relative to the suite folder. */
+  file?: string
+}
+
+type Call = { method?: string; url?: string; request?: { headers?: Record<string, string>; body?: string }; response?: { status?: number; body?: string } }
+
+const MAX_ACTIONS = 15
+const MAX_BODY = 600
+
+const clip = (s: string, n: number) => (s.length > n ? `${s.slice(0, n - 1)}…` : s)
+const quote = (s: string) => `'${s.replace(/'/g, `'\\''`)}'`
+
+/** The request a "request" defect is about, from the run's network.json: the last call to its route with its status. */
+export function failingRequest(runDir: string | undefined, d: CaseDefect): FailingRequest | undefined {
+  if (d.kind !== "request" || !runDir) return undefined
+  const file = path.join(runDir, "network.json")
+  if (!existsSync(file)) return undefined
+  let calls: Call[] = []
+  try {
+    calls = (JSON.parse(readFileSync(file, "utf8")) as { calls?: Call[] }).calls || []
+  } catch {
+    return undefined
+  }
+  const hit = [...calls].reverse().find((c) => c.url && (c.response?.status || 0) >= 400 && d.title.startsWith(`${routeOf(c.url)} `))
+  if (!hit?.url) return undefined
+  const method = (hit.method || "GET").toUpperCase()
+  const headers = Object.entries(hit.request?.headers || {}).map(([k, v]) => `-H ${quote(`${k}: ${v}`)}`)
+  const body = hit.request?.body
+  const curl = ["curl -i", ...(method === "GET" ? [] : [`-X ${method}`]), quote(hit.url), ...headers, ...(body ? [`--data-raw ${quote(body)}`] : [])].join(" ")
+  return {
+    method,
+    url: hit.url,
+    ...(hit.response?.status ? { status: hit.response.status } : {}),
+    ...(body ? { requestBody: clip(body, MAX_BODY) } : {}),
+    ...(hit.response?.body ? { responseBody: clip(hit.response.body, MAX_BODY) } : {}),
+    curl,
+  }
+}
+
+/** The browser's actions that worked, without repeats in a row and without the agent's own bookkeeping. */
+function browserActions(runDir?: string): string[] {
+  const out: string[] = []
+  for (const t of readTimeline(runDir)) {
+    for (const a of t.actions || []) {
+      if (!a.ok || a.done || !a.text.trim() || /^(done|wait\b)/i.test(a.text)) continue
+      if (out[out.length - 1] !== a.text) out.push(a.text)
+    }
+  }
+  return out.length > MAX_ACTIONS ? [...out.slice(0, MAX_ACTIONS - 1), `… and ${out.length - MAX_ACTIONS + 1} more (steps.json)`] : out
+}
+
+/** How sure it is that the defect happens again: a second run of it, else its history, else seen once. */
+export function reproducibility(r: CaseResult, seen = 0, intermittent?: { failed: number; runs: number }): { reproducibility: Reproducibility; detail: string } {
+  if (intermittent) return { reproducibility: "intermittent", detail: `its case failed in ${intermittent.failed} of the last ${intermittent.runs} runs` }
+  if (r.reproduced) {
+    const { runs, failed } = r.reproduced
+    return failed >= runs
+      ? { reproducibility: "always", detail: `failed the same way in ${failed} of ${runs} runs` }
+      : { reproducibility: "intermittent", detail: `failed the same way in ${failed} of ${runs} runs` }
+  }
+  if (seen) return { reproducibility: "always", detail: `seen in ${seen} earlier run${seen === 1 ? "" : "s"} too` }
+  return { reproducibility: "once", detail: "seen once; not run again" }
+}
+
+/** The repro pack of a defect, from the run of one of its cases (the first). `suiteDir` makes file paths relative. */
+export function reproPack(r: CaseResult, d: CaseDefect, opts: { env?: ReportEnv; seen?: number; intermittent?: { failed: number; runs: number }; suiteDir?: string } = {}): ReproPack {
+  const verdicts = stepVerdictLines(r)
+  const steps = verdicts.length ? verdicts : r.case.steps.map((s, i) => `${i + 1}. ${s.action.trim()}${s.data ? ` (data: ${s.data})` : ""}${s.expected ? ` — expected: ${s.expected}` : ""}`)
+  const e = opts.env
+  const env = [
+    ...(e?.browser ? [`Browser: ${e.browser}${r.device ? `, device ${r.device}` : ""}`] : r.device ? [`Device: ${r.device}`] : []),
+    ...(e?.ci ? [`Build: ${e.ci.name}${e.ci.url ? ` ${e.ci.url}` : ""}`] : []),
+    ...(e?.commit ? [`Commit: ${e.commit}${e.branch ? ` on ${e.branch}` : ""}`] : []),
+    ...(e?.version ? [`ForgeQE ${e.version} on ${e.machine}`] : []),
+    `When: ${r.finishedAt.slice(0, 16).replace("T", " ")} UTC`,
+  ]
+  const rel = (f: string) => (opts.suiteDir ? path.relative(opts.suiteDir, f) : f)
+  const files: ReproPack["files"] = []
+  if (r.runDir) {
+    for (const [name, what] of [["failure.png", "Last screenshot"], ["workflow.yaml", "Replay flow (forgeqe replay)"], ["steps.json", "Every browser step"], ["network.json", "Every API call, secrets removed"], ["report.html", "The run's report"]]) {
+      const f = path.join(r.runDir, name)
+      if (existsSync(f)) files.push({ name: what, path: rel(f) })
+    }
+  }
+  if (r.code?.knownDefect && r.code.testFile) files.push({ name: "Test that asserts the correct behaviour (an expected failure)", path: r.code.testFile })
+  const request = failingRequest(r.runDir, d)
+  return {
+    ...reproducibility(r, opts.seen, opts.intermittent),
+    steps,
+    actions: browserActions(r.runDir),
+    ...(r.case.startUrl || e?.startUrl ? { startUrl: r.case.startUrl || e?.startUrl } : {}),
+    ...(r.finalUrl ? { endUrl: r.finalUrl } : {}),
+    at: r.finishedAt,
+    env,
+    ...(request ? { request } : {}),
+    files,
+  }
+}
+
+/** The pack as plain text, for a bug's description: reproducibility, the steps, and the failing request. */
+export function reproText(p: ReproPack): string {
+  const lines = [`Reproducibility: ${p.reproducibility} (${p.detail}).`]
+  if (p.startUrl) lines.push(`Start at: ${p.startUrl}`)
+  if (p.actions.length) lines.push("", "What the browser did:", ...p.actions.map((a, i) => `${i + 1}. ${a}`))
+  if (p.request) {
+    lines.push("", `Failing request: ${p.request.method} ${p.request.url}${p.request.status ? ` → ${p.request.status}` : ""}`, "Run it again (secrets already removed):", p.request.curl)
+    if (p.request.responseBody) lines.push(`Response: ${p.request.responseBody}`)
+  }
+  if (p.env.length) lines.push("", "Environment:", ...p.env.map((l) => `- ${l}`))
+  return lines.join("\n")
+}
+
+const fence = (s: string) => `\`\`\`\n${s}\n\`\`\``
+
+/** defects/<id>.md in the suite folder, one per defect with a pack: everything to make it happen again, and the analysis. */
+export function writeReproFiles(suiteDir: string, clusters: DefectCluster[]): void {
+  const dir = path.join(suiteDir, "defects")
+  for (const c of clusters) {
+    const p = c.repro
+    if (!p) continue
+    const rca = c.rca
+    const md = [
+      `# ${c.title}`,
+      "",
+      `${c.failureLabel} · ${c.cases.length} case${c.cases.length === 1 ? "" : "s"} · reproducibility: **${p.reproducibility}** (${p.detail})${c.bug ? ` · bug ${c.bug.url ? `[${c.bug.label}](${c.bug.url})` : c.bug.label}` : ""}`,
+      "",
+      "## Cases that fail this way",
+      ...c.cases.map((x) => `- ${x.id} ${x.title} (${x.status})`),
+      "",
+      "## Evidence",
+      ...c.evidence.map((e) => `- ${e}`),
+      "",
+      "## To reproduce",
+      ...(p.startUrl ? [`Start at ${p.startUrl}.`, ""] : []),
+      ...p.steps.map((s) => `- ${s}`),
+      ...(p.actions.length ? ["", "What the browser did:", ...p.actions.map((a, i) => `${i + 1}. ${a}`)] : []),
+      ...(p.endUrl ? ["", `It ended on ${p.endUrl}.`] : []),
+      ...(p.request
+        ? ["", "## Failing request", `${p.request.method} ${p.request.url}${p.request.status ? ` → ${p.request.status}` : ""}. Secrets were removed when the run recorded it.`, "", fence(p.request.curl), ...(p.request.requestBody ? ["", "Request body:", fence(p.request.requestBody)] : []), ...(p.request.responseBody ? ["", "Response body:", fence(p.request.responseBody)] : [])]
+        : []),
+      "",
+      "## Environment",
+      ...p.env.map((l) => `- ${l}`),
+      ...(p.files.length ? ["", "## Files", ...p.files.map((f) => `- ${f.name}: \`${f.path}\``)] : []),
+      ...(rca ? ["", "## Likely cause (a hypothesis, not a verdict)", ...rcaLines(rca)] : []),
+      "",
+    ].join("\n")
+    mkdirSync(dir, { recursive: true })
+    const name = `${c.id}.md`
+    writeFileSync(path.join(dir, name), md)
+    p.file = `defects/${name}`
+  }
+}
+
+/** The analysis as markdown lines, for the repro file and the bug comment. */
+export function rcaLines(rca: NonNullable<DefectCluster["rca"]>): string[] {
+  return [
+    `- Layer: ${rca.layer}${rca.component ? ` · component: ${rca.component}` : ""} (${rca.basis})`,
+    ...(rca.hypothesis ? [`- Hypothesis (${rca.confidence || "low"} confidence): ${rca.hypothesis}`] : []),
+    ...(rca.evidence?.length ? ["- Based on:", ...rca.evidence.map((e) => `  - ${e}`)] : []),
+    ...(rca.confirm?.length ? ["- To confirm:", ...rca.confirm.map((e) => `  - ${e}`)] : []),
+    ...(rca.files?.length ? [`- Code to look at: ${rca.files.join(", ")}`] : []),
+    ...(rca.suspects?.length ? [`- Suspect commits (${rca.range}):`, ...rca.suspects.map((s) => `  - ${s.sha.slice(0, 10)} ${s.subject} (${s.author}, ${s.at.slice(0, 10)}): ${s.why}`)] : rca.range ? [`- No commit ${rca.range} mentions it.`] : []),
+    ...(rca.note ? [`- ${rca.note}`] : []),
+    ...(rca.model ? [`- Written by ${rca.model}${rca.tokens ? `, ${rca.tokens} tokens` : ""}${rca.cost ? `, ${rca.cost.amount.toFixed(2)} ${rca.cost.currency}` : ""}.`] : []),
+  ]
+}

package/src/cases/types.ts

@@ -1,10 +1,11 @@
 import type { CodeStatus, StepVerdict, Usage } from "../types.ts"
 import type { Cost } from "../usage.ts"
+import type { CaseDefect, Reproduced } from "./defects.ts"
 import type { Failure } from "./failure.ts"
 
 export type CaseStep = { action: string; data?: string; expected?: string }
 
-export type CaseSource = "text" | "file" | "ado" | "jira" | "xray" | "testrail" | "zephyr" | "qtest"
+export type CaseSource = "text" | "file" | "ado" | "jira" | "xray" | "testrail" | "zephyr" | "qtest" | "github"
 
 /** A test case from any source, in one shape. It becomes one browser goal and, once it passes, one test. */
 export type TestCase = {
@@ -33,6 +34,8 @@ export type CaseStatus = "passed" | "failed" | "stopped" | "error"
 export type CaseResult = {
   case: TestCase
   status: CaseStatus
+  /** The named device this result ran on (browser_devices); its case id and title carry the device too. */
+  device?: string
   /** The browser agent's account of the run, or why it could not run. */
   summary: string
   startedAt: string
@@ -73,6 +76,10 @@ export type CaseResult = {
   codeSkipped?: string
   /** For a case that did not pass: what kind of failure it was, and so what happened next (see src/cases/failure.ts). */
   failure?: Failure
+  /** For a case that failed or could not run: the defect it belongs to, shared with every case failing the same way. */
+  defect?: CaseDefect
+  /** The case was run once more to see whether its defect repeats (the first case of each new defect). */
+  reproduced?: Reproduced
   /** The bug filed (or found open) for a functional defect, when bugs are set up. */
   bug?: { label: string; url?: string }
   /** The browser run was repeated once because the first one met an environment problem. */

package/src/cli/help.ts

@@ -14,7 +14,7 @@ Usage:
   forgeqe features [--config forgeqe.yaml]   what ForgeQE can do, and what each capability still needs here
   forgeqe docs [--guide|--open] [--copy DIR]  the guides and the architecture page (--guide: the user guide in the browser; --open: the architecture page)
   forgeqe start [--profile name] [--url URL] [--goal TEXT] [--headed] [--replay MODE] [--no-memory]
-                 [--repo DIR] [--no-code]
+                 [--repo DIR] [--no-code] [--browser-provider local|browserstack|testmu] [--devices a,b]
   forgeqe run "<request>" [--profile name] [--url URL] [--headed] [--plain] [--max-steps N]
                [--repo DIR] [--no-code] [--replay MODE] [--no-memory] [--attempts N] [--code-model provider/model]
                [--browser-provider local|browserstack|testmu]
@@ -23,7 +23,7 @@ Usage:
   forgeqe run --direct "<goal>" [--out DIR] [--skill NAME] [--browser-provider local|browserstack|testmu] ...  one browser goal, no agent
   forgeqe suite <source...> [--parallel N] [--repo DIR] [--no-code] [--no-write-back] [--out DIR]
                  [--profile name] [--url URL] [--headed] [--plain] [--attempts N] [--skill NAME]
-                 [--browser-provider local|browserstack|testmu]
+                 [--browser-provider local|browserstack|testmu] [--devices a,b]
                                                run test cases the same way every time (what pipelines call)
   forgeqe cases <source...> [--goal]          list the test cases a source gives (--goal: the goal each runs)
   forgeqe code [run-dir] [--repo DIR] [--attempts N] [--code-model provider/model]
@@ -38,15 +38,20 @@ Usage:
   forgeqe mcp [--repo DIR] [--profile name] [--url URL] [--headed] [--browser-provider local|browserstack|testmu]
                                                ForgeQE's tools as an MCP server on stdio, for other agents
   forgeqe mcp list                            the Azure DevOps, Jira and other MCP servers ForgeQE's agent gets
-  forgeqe bi [build] [--evidence-dir DIR] [--data-dir DIR] [--build-dir DIR]
-                                               export ForgeQE + ADO/Jira datasets into an Evidence project and build static BI HTML
-  forgeqe bi init                             scaffold or refresh the managed Evidence project files
+  forgeqe bi [build] [--evidence-dir DIR] [--data-dir DIR] [--build-dir DIR] [--static | --evidence]
+                                               export ForgeQE + ADO/Jira datasets into an Evidence project and build a static
+                                               site; without Node.js or when Evidence fails, ForgeQE's own page (--static: always,
+                                               --evidence: fail instead)
+  forgeqe bi init                             scaffold the Evidence project and install its packages
   forgeqe bi sync                             refresh the CSV datasets and generated Evidence pages, without building
   forgeqe pr-tests [pr] [--base BRANCH] [--agent] [--run] [--post]
                                                the tests a pull request (or the commits since BRANCH) can break; --run runs
                                                them and exits 1 when one fails (a CI gate), --post puts the results on the PR
   forgeqe quality [run-dir]                   a run's WCAG rules passed and failed, Core Web Vitals, and budgets
   forgeqe quality trends [page]               accessibility and timings per page over time: what got worse or better
+  forgeqe defects [--folder DIR]              defects over the saved suites: still there, gone, came back, time to fix, by area
+  forgeqe defects predict [--repo DIR] [--base REF] [--head REF]
+                                               where a change is likely to cause defects, with reasons and the cases to run first
   forgeqe notify test [channel]                  send a test card to the Teams and email notification channels
   forgeqe eval domain [--only id,id] [--min 0.8]  does the agent apply domain and regulatory knowledge? (with the domain declared and without)
   forgeqe prompts [list]                      every agent prompt and where it comes from (ForgeQE's or your edited copy)
@@ -110,6 +115,9 @@ page with ranked locators and a confidence %, plus the steps of each goal's last
 --replay trust    replay and finish without the LLM when every step works
 --replay off      never replay; still record pages
 --browser-provider  pick which configured browser provider runs this command's browser work
+--devices a,b       run on each named device (devices: in the BrowserStack/TestMu YAML file; in the shell, describe
+                    them and ForgeQE's agent saves them): devices run side by side, each reported on its own, and each
+                    case posts one combined result back (it fails when any device fails)
 
 ForgeQE's agent (forgeqe start, forgeqe run) runs on the agent runtime with the active profile's model (or code.model) and
 calls ForgeQE's tools: browser runs, reading and running test cases, writing tests and running them, page memory, pushing
@@ -138,8 +146,9 @@ The report is report.html (step verdicts, timeline with screenshots, timings, en
 out/suites/<time>/; every run folder also has its own report.html.
 
 Static BI (bi: in forgeqe.yaml): \`forgeqe bi\` writes curated CSV datasets from ForgeQE's own suite and readiness
-reports plus optional Azure DevOps WIQL and Jira JQL datasets, scaffolds an Evidence project, and runs
-\`evidence build\` to produce static HTML in the build directory. No server stays up between runs.
+reports plus optional Azure DevOps WIQL and Jira JQL datasets, scaffolds an Evidence project (@evidence-dev/evidence
+40.x, needs Node.js), and runs \`evidence sources\` and \`evidence build\` to produce a static site in the build directory;
+when that cannot run, it writes ForgeQE's own self-contained page instead. No server stays up between runs.
 
 Healing (forgeqe heal): ForgeQE runs the tests (the command given, code.test_command, or one the agent finds).
 For each failing test, Scribe re-runs its flow in the real browser: when the flow still works, the test is out

package/src/cli/index.ts

@@ -10,7 +10,7 @@ import { goalFor, loadCases } from "../cases/index.ts"
 import { codeSkipReason, latestRunDir, runCodeAgent } from "../code/index.ts"
 import { detailFrom } from "../detail.ts"
 import { ensureOpencode, INSTALL_HINT } from "../code/runtime.ts"
-import { browserProvider, loadConfig, parseDuration, replayMode, selectProfile, userConfigDir, type AppConfig, type BrowserProvider } from "../config.ts"
+import { browserConfigFor, browserProvider, loadConfig, parseDuration, replayMode, selectProfile, userConfigDir, type AppConfig, type BrowserProvider } from "../config.ts"
 import { runHeal } from "../heal/loop.ts"
 import { formatHeal } from "../heal/report.ts"
 import { backgroundMonitors, stopBackgroundMonitors } from "../monitor/background.ts"
@@ -30,7 +30,7 @@ import { externalMcp } from "../mcp/external.ts"
 import { createMcpServer, serveStdio } from "../mcp/server.ts"
 import { TOOLS, environmentNotes, type ToolContext } from "../mcp/tools.ts"
 import { formatSuite } from "../suite/report.ts"
-import { runSuite, suiteExitCode } from "../suite/run.ts"
+import { runSuite, suiteExitCode, suiteParallel } from "../suite/run.ts"
 import { packageVersion } from "../tui/theme.ts"
 import { update } from "../update.ts"
 import { meterSnapshot, requestLine } from "../usage.ts"
@@ -67,6 +67,12 @@ function browserProviderFlag(flags: Flags): BrowserProvider | undefined {
   return value ? browserProvider(value) : undefined
 }
 
+/** --devices win11-chrome,mac-edge: run on each named device (browser_devices, or devices: in a provider file). */
+function devicesFlag(flags: Flags): string[] | undefined {
+  const names = (strFlag(flags, "devices") || "").split(",").map((d) => d.trim()).filter(Boolean)
+  return names.length ? [...new Set(names)] : undefined
+}
+
 function debugRequested(flags: Flags): boolean {
   return boolFlag(flags, "debug") || /^(1|true|yes|on)$/i.test(process.env.FORGEQE_DEBUG || "")
 }
@@ -217,10 +223,23 @@ export async function main(argv: string[]): Promise<number> {
         const cfg = loadConfig(configOpts(flags))
         const trends = positionals[1] === "trends"
         const color = Boolean(process.stdout.isTTY) && !process.env.NO_COLOR
-    const ctx: ToolContext = { config: cfg, cwd: process.cwd(), session: "once", emit: toolPrinter(color, (line) => console.log(line), detailOf(flags)) }
+        const ctx: ToolContext = { config: cfg, cwd: process.cwd(), session: "once", emit: toolPrinter(color, (line) => console.log(line), detailOf(flags)) }
         const out = (await TOOLS.find((t) => t.name === (trends ? "quality_trends" : "quality_report"))!.run(trends ? { page: positionals[2] } : { run_dir: positionals[1] }, ctx)) as { budget_exceeded?: unknown[]; pages?: { slower: unknown[]; newly_failing_rules: unknown[] }[] }
         return out.budget_exceeded?.length || out.pages?.some((p) => p.slower.length || p.newly_failing_rules.length) ? 1 : 0
       }
+      case "defects": {
+        // forgeqe defects [--folder DIR]: trends; forgeqe defects predict [--repo DIR] [--base REF] [--head REF].
+        const cfg = loadConfig(configOpts(flags))
+        const color = Boolean(process.stdout.isTTY) && !process.env.NO_COLOR
+        const ctx: ToolContext = { config: cfg, cwd: process.cwd(), session: "once", emit: toolPrinter(color, (line) => console.log(line), detailOf(flags)) }
+        const str = (k: string) => (typeof flags.get(k) === "string" ? String(flags.get(k)) : undefined)
+        if (positionals[1] === "predict") {
+          await TOOLS.find((t) => t.name === "predict_defects")!.run({ repo: str("repo"), base: str("base"), head: str("head"), folder: str("folder") }, ctx)
+          return 0
+        }
+        await TOOLS.find((t) => t.name === "defect_trends")!.run({ folder: str("folder") }, ctx)
+        return 0
+      }
       case "logs":
         return cmdLogs(positionals.slice(1), flags)
       case "skills":
@@ -382,7 +401,7 @@ async function cmdApi(args: string[], flags: Flags): Promise<number> {
   const cfg = loadConfig(configOpts(flags))
   const action = args[0] || "list"
   const color = Boolean(process.stdout.isTTY) && !process.env.NO_COLOR
-    const ctx: ToolContext = { config: cfg, cwd: process.cwd(), session: "once", emit: toolPrinter(color, (line) => console.log(line), detailOf(flags)) }
+  const ctx: ToolContext = { config: cfg, cwd: process.cwd(), session: "once", emit: toolPrinter(color, (line) => console.log(line), detailOf(flags)) }
   const tool = (name: string) => TOOLS.find((t) => t.name === name)!
   const csv = (name: string) => (strFlag(flags, name) || "").split(",").map((s) => s.trim()).filter(Boolean)
   switch (action) {
@@ -489,11 +508,16 @@ async function cmdBi(args: string[], flags: Flags): Promise<number> {
     dataDir: strFlag(flags, "data-dir"),
     buildDir: strFlag(flags, "build-dir") || strFlag(flags, "out"),
   }
+  const shown = (file: string) => path.relative(process.cwd(), file) || file
+  const printNotes = (notes: string[]) => {
+    for (const note of notes) console.log(`note              ${note}`)
+  }
   switch (action) {
     case "init": {
       const out = await initBiProject(cfg, overrides)
       console.log(`Evidence project  ${out.evidenceDir}`)
-      for (const file of out.files) console.log(`generated         ${path.relative(process.cwd(), file) || file}`)
+      for (const file of out.files) console.log(`generated         ${shown(file)}`)
+      printNotes(out.notes)
       return 0
     }
     case "sync": {
@@ -505,23 +529,30 @@ async function cmdBi(args: string[], flags: Flags): Promise<number> {
       console.log(`Case results      ${out.caseResults}`)
       console.log(`Readiness reports ${out.readinessReports}`)
       console.log(`External datasets ${out.externalDatasets}`)
-      console.log(`Summary           ${path.relative(process.cwd(), out.summaryFile) || out.summaryFile}`)
+      console.log(`Summary           ${shown(out.summaryFile)}`)
+      printNotes(out.notes)
       return 0
     }
     case "build": {
-      const out = await buildBi(cfg, overrides)
+      const out = await buildBi(cfg, overrides, boolFlag(flags, "static") ? "static" : boolFlag(flags, "evidence") ? "evidence" : "auto")
       console.log(`Evidence project  ${out.evidenceDir}`)
       console.log(`Static site       ${out.buildDir}`)
-      console.log(`Index             ${path.relative(process.cwd(), out.indexFile) || out.indexFile}`)
-      console.log(`Summary           ${path.relative(process.cwd(), out.summaryFile) || out.summaryFile}`)
+      console.log(`Index             ${shown(out.indexFile)}`)
+      console.log(`Summary           ${shown(out.summaryFile)}`)
       console.log(`Suite runs        ${out.suites}`)
       console.log(`Case results      ${out.caseResults}`)
       console.log(`Readiness reports ${out.readinessReports}`)
       console.log(`External datasets ${out.externalDatasets}`)
+      if (out.engine === "evidence") {
+        console.log("Built with        Evidence. Serve the folder to view it (npx serve, or evidence preview): opened from disk it cannot load its data.")
+      } else {
+        console.log(`Built with        ForgeQE's own page, without Evidence (${out.fallbackReason}). It opens straight from disk.`)
+      }
+      printNotes(out.notes)
       return 0
     }
     default:
-      throw new Error("Usage: forgeqe bi [init|sync|build] [--evidence-dir DIR] [--data-dir DIR] [--build-dir DIR]")
+      throw new Error("Usage: forgeqe bi [init|sync|build] [--evidence-dir DIR] [--data-dir DIR] [--build-dir DIR] [--static | --evidence]")
   }
 }
 
@@ -784,6 +815,8 @@ async function cmdStart(flags: Flags): Promise<number> {
     url: strFlag(flags, "url"),
     goal: strFlag(flags, "goal"),
     headed: boolFlag(flags, "headed"),
+    browserProvider: browserProviderFlag(flags),
+    devices: devicesFlag(flags),
     verbose: boolFlag(flags, "verbose"),
     ...memoryFlags(flags),
     repo: boolFlag(flags, "no-code") ? null : strFlag(flags, "repo") ? path.resolve(strFlag(flags, "repo")!) : undefined,
@@ -929,6 +962,7 @@ async function runAgent(cfg: AppConfig, flags: Flags, text: string, maxSteps: nu
       context: () => ({
         headed: boolFlag(flags, "headed"),
         browserProvider: browserProviderFlag(flags),
+        devices: devicesFlag(flags),
         maxSteps,
         url: strFlag(flags, "url"),
         ...memoryFlags(flags),
@@ -962,7 +996,7 @@ async function runAgent(cfg: AppConfig, flags: Flags, text: string, maxSteps: nu
 }
 
 function cmdMcpList(cfg: AppConfig): number {
-  const { servers, skipped } = externalMcp(cfg)
+  const { servers, skipped } = externalMcp(cfg, Bun.which, process.env, { cwd: process.cwd() })
   if (!servers.length && !skipped.length) {
     console.log("No MCP servers for other systems. Set sources.azure_devops or sources.jira, or add servers under mcp:, in forgeqe.yaml.")
   }
@@ -994,6 +1028,7 @@ async function cmdMcp(args: string[], flags: Flags): Promise<number> {
     cwd,
     headed: boolFlag(flags, "headed"),
     browserProvider: browserProviderFlag(flags),
+    devices: devicesFlag(flags),
     maxSteps: maxStepsFlag(flags),
     url: strFlag(flags, "url"),
     ...memoryFlags(flags),
@@ -1011,10 +1046,13 @@ async function runGoal(cfg: AppConfig, flags: Flags, goal: string, maxSteps: num
   const out = runDirectory(cfg.export.outDir, strFlag(flags, "out"))
   const t0 = Date.now()
   let steps = 0
+  const devices = devicesFlag(flags) || []
+  if (devices.length > 1) throw new Error("run --direct runs one browser: pass one device, or use forgeqe suite or ForgeQE's agent for several.")
   const { result, files } = await executeRun({
     config: cfg,
     goal,
-    browserProvider: browserProviderFlag(flags),
+    browserProvider: devices.length ? undefined : browserProviderFlag(flags),
+    browserDevice: devices[0],
     url: strFlag(flags, "url"),
     headed: boolFlag(flags, "headed"),
     maxSteps,
@@ -1253,11 +1291,13 @@ async function cmdSuite(refs: string[], flags: Flags): Promise<number> {
   cfg = withCodeFlags(cfg, flags)
   const parallelRaw = strFlag(flags, "parallel")
   if (parallelRaw && !(Number(parallelRaw) >= 1)) throw new Error("--parallel must be a positive number")
-  const parallel = parallelRaw ? Number(parallelRaw) : cfg.suite.parallel
   const cases = await loadCases(refs, cfg.sources)
+  const devices = devicesFlag(flags) || []
+  for (const d of devices) browserConfigFor(cfg, undefined, d)
+  const parallel = suiteParallel(parallelRaw ? Number(parallelRaw) : undefined, cfg.suite.parallel, devices.length, cases.length * Math.max(1, devices.length))
   const repo = codeRepo(cfg, flags)
   const color = Boolean(process.stdout.isTTY) && !boolFlag(flags, "plain")
-  console.log(`${cases.length} test case${cases.length === 1 ? "" : "s"}${parallel > 1 ? `, ${parallel} at a time` : ""}${repo ? `; passed cases get a test in ${repo}` : ""}`)
+  console.log(`${cases.length} test case${cases.length === 1 ? "" : "s"}${devices.length ? ` on ${devices.length} device${devices.length === 1 ? "" : "s"} (${devices.join(", ")})` : ""}${parallel > 1 ? `, ${parallel} at a time` : ""}${repo ? `; passed cases get a test in ${repo}` : ""}`)
 
   const controller = new AbortController()
   const onSigint = () => controller.abort()
@@ -1267,7 +1307,8 @@ async function cmdSuite(refs: string[], flags: Flags): Promise<number> {
       config: cfg,
       cases,
       repo,
-      browserProvider: browserProviderFlag(flags),
+      browserProvider: devices.length ? undefined : browserProviderFlag(flags),
+      devices,
       parallel,
       outDir: strFlag(flags, "out"),
       writeBack: boolFlag(flags, "no-write-back") ? false : undefined,