@capagents/forgeqe 0.0.0-stage → 1.0.0

package/src/monitor/run.ts

@@ -0,0 +1,263 @@
+import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, unlinkSync, writeFileSync } from "node:fs"
+import path from "node:path"
+
+import type { Card } from "../card.ts"
+import { loadCases, type TestCase } from "../cases/index.ts"
+import type { AppConfig } from "../config.ts"
+import { counts, suiteUsage } from "../suite/report.ts"
+import { runSuite, type SuiteEvent, type SuiteOptions, type SuiteResult } from "../suite/run.ts"
+import { formatDuration, formatTokens } from "../tui/theme.ts"
+import { usageLine } from "../usage.ts"
+import { alertTitle, itemLine, sendAlerts, type AlertMessage, type Sent } from "./alerts.ts"
+import { MONITOR_CHECK, decide, readState, writeState, type AlertItem, type CheckOutcome, type CheckState } from "./state.ts"
+
+export type RoundResult = {
+  at: string
+  suite?: SuiteResult
+  /** Why the round could not run at all. */
+  error?: string
+  alerts: AlertItem[]
+  sent: Sent[]
+  /** Checks failing after this round, alerted or not. */
+  failing: (CheckState & { id: string })[]
+  failAfter: number
+  stopped: boolean
+}
+
+export type MonitorOptions = {
+  config: AppConfig
+  /** Test case sources; default monitor.checks. */
+  checks: string[]
+  headed?: boolean
+  url?: string
+  /** Browser skills for every check (empty: none); unset: the engine picks once per check for the monitor's lifetime. */
+  skills?: string[]
+  signal?: AbortSignal
+  onEvent?: (e: SuiteEvent) => void
+  /** Replace loading cases, running them, and sending alerts; tests use them. */
+  load?: (refs: string[], cfg: AppConfig) => Promise<TestCase[]>
+  execute?: SuiteOptions["execute"]
+  send?: typeof sendAlerts
+  now?: () => Date
+}
+
+/** The round's report as a link: under monitor.report_url when set, else the local file. */
+function reportLinks(cfg: AppConfig, suite?: SuiteResult): { report?: string; reportUrl?: string } {
+  if (!suite) return {}
+  const base = cfg.monitor.reportUrl
+  return { report: suite.files.html, ...(base ? { reportUrl: `${base}/${path.basename(suite.dir)}/report.html` } : {}) }
+}
+
+/**
+ * One monitoring round: run every check as a suite case (replaying page memory, so a known flow needs one model
+ * call to confirm the outcome, or none with replay trust), update the saved state, and alert on what changed.
+ */
+export async function monitorRound(opts: MonitorOptions): Promise<RoundResult> {
+  const { config } = opts
+  const m = config.monitor
+  const now = opts.now || (() => new Date())
+  const at = now()
+  const prev = readState(m.outDir)
+  const policy = { failAfter: m.failAfter, remindMs: m.remindMs }
+  let suite: SuiteResult | undefined
+  let error: string | undefined
+  try {
+    const cases = await (opts.load || ((refs, cfg) => loadCases(refs, cfg.sources)))(opts.checks, config)
+    if (!cases.length) throw new Error(`No checks found in ${opts.checks.join(" ")}.`)
+    suite = await runSuite({
+      config: { ...config, suite: { ...config.suite, outDir: m.outDir, code: false } },
+      cases,
+      parallel: Math.min(m.parallel, cases.length),
+      writeBack: m.writeBack,
+      replay: m.replay,
+      headed: opts.headed,
+      url: opts.url,
+      skills: opts.skills,
+      signal: opts.signal,
+      onEvent: opts.onEvent,
+      execute: opts.execute,
+    })
+  } catch (err) {
+    error = (err as Error).message || String(err)
+  }
+
+  const stopped = Boolean(opts.signal?.aborted)
+  const finished = now()
+  let decision: ReturnType<typeof decide>
+  if (suite) {
+    const outcomes: CheckOutcome[] = suite.results.map((r) => ({ id: r.case.id, title: r.case.title, status: r.status, summary: r.summary, result: r }))
+    outcomes.push({ ...MONITOR_CHECK, status: "passed", summary: "" })
+    decision = decide(prev, outcomes, finished, policy)
+  } else {
+    const status = stopped ? "stopped" : "error"
+    decision = decide(prev, [{ ...MONITOR_CHECK, status, summary: `The round could not run: ${error}` }], finished, policy, true)
+  }
+  writeState(m.outDir, decision.state)
+
+  let sent: Sent[] = []
+  if (decision.alerts.length) {
+    const c = suite ? counts(suite.results) : undefined
+    const message: AlertMessage = {
+      title: alertTitle(decision.alerts),
+      items: decision.alerts,
+      at: at.toISOString(),
+      passed: c?.passed ?? 0,
+      total: c?.total ?? 0,
+      ...reportLinks(config, suite),
+    }
+    sent = await (opts.send || sendAlerts)(m.alerts, message)
+    if (suite) writeFileSync(path.join(suite.dir, "alerts.json"), `${JSON.stringify({ title: message.title, alerts: decision.alerts.map(({ result, ...a }) => a), sent }, null, 2)}\n`)
+  }
+  pruneRounds(m.outDir, m.keep)
+  const failing = Object.entries(decision.state.checks)
+    .filter(([, s]) => s.status === "failing")
+    .map(([id, s]) => ({ id, ...s }))
+  return { at: at.toISOString(), suite, error, alerts: decision.alerts, sent, failing, failAfter: m.failAfter, stopped }
+}
+
+/** Delete the oldest round folders (those with a suite.json) beyond `keep`. */
+export function pruneRounds(outDir: string, keep: number): string[] {
+  if (!existsSync(outDir)) return []
+  const rounds = readdirSync(outDir)
+    .filter((n) => existsSync(path.join(outDir, n, "suite.json")))
+    .sort()
+  const old = rounds.slice(0, Math.max(0, rounds.length - keep))
+  for (const n of old) rmSync(path.join(outDir, n), { recursive: true, force: true })
+  return old
+}
+
+/** 0 when every check passed, 2 when stopped, otherwise 1. */
+export function roundExitCode(r: RoundResult): number {
+  if (r.stopped) return 2
+  if (!r.suite) return 1
+  return r.suite.results.every((x) => x.status === "passed") ? 0 : 1
+}
+
+/** One monitor per folder: a second one (or an overlapping cron run) would race on state.json. */
+export function lockMonitor(outDir: string): () => void {
+  mkdirSync(outDir, { recursive: true })
+  const file = path.join(outDir, ".monitor.lock")
+  const take = () => writeFileSync(file, String(process.pid), { flag: "wx" })
+  try {
+    take()
+  } catch {
+    const pid = Number(readFileSync(file, "utf8").trim())
+    let alive = false
+    try {
+      alive = pid > 0 && pid !== process.pid && process.kill(pid, 0)
+    } catch {}
+    if (alive) throw new Error(`Another forgeqe monitor (pid ${pid}) is using ${outDir}. Stop it, or set monitor.out_dir to another folder.`)
+    unlinkSync(file)
+    take()
+  }
+  return () => {
+    try {
+      if (readFileSync(file, "utf8").trim() === String(process.pid)) unlinkSync(file)
+    } catch {}
+  }
+}
+
+function sleep(ms: number, signal?: AbortSignal): Promise<void> {
+  return new Promise((resolve) => {
+    if (signal?.aborted) return resolve()
+    const timer = setTimeout(done, ms)
+    function done() {
+      clearTimeout(timer)
+      signal?.removeEventListener("abort", done)
+      resolve()
+    }
+    signal?.addEventListener("abort", done, { once: true })
+  })
+}
+
+/**
+ * Run rounds until stopped: each starts monitor.every after the previous one started, or straight after it when a
+ * round overruns. With `once`, one round and its exit code.
+ */
+export async function runMonitor(opts: MonitorOptions & { once?: boolean; onRound?: (r: RoundResult, nextAt?: Date) => void; wait?: typeof sleep }): Promise<number> {
+  const unlock = lockMonitor(opts.config.monitor.outDir)
+  const wait = opts.wait || sleep
+  try {
+    if (opts.once) {
+      const r = await monitorRound(opts)
+      opts.onRound?.(r)
+      return roundExitCode(r)
+    }
+    while (!opts.signal?.aborted) {
+      const start = Date.now()
+      const r = await monitorRound(opts)
+      if (opts.signal?.aborted) {
+        opts.onRound?.(r)
+        break
+      }
+      const nextAt = new Date(Math.max(Date.now(), start + opts.config.monitor.everyMs))
+      opts.onRound?.(r, nextAt)
+      await wait(nextAt.getTime() - Date.now(), opts.signal)
+    }
+    return 0
+  } finally {
+    unlock()
+  }
+}
+
+const CHANNEL = { teams: "Teams", email: "email" } as const
+
+function hhmm(iso: string | Date): string {
+  const d = new Date(iso)
+  return `${String(d.getHours()).padStart(2, "0")}:${String(d.getMinutes()).padStart(2, "0")}`
+}
+
+/** A round in the terminal: the result line, what changed, alerts sent, the report, and when the next round is. */
+export function formatRound(r: RoundResult, nextAt?: Date, cwd = process.cwd()): string {
+  const lines: string[] = []
+  if (!r.suite) lines.push(`[${hhmm(r.at)}] ${r.stopped ? "■ Stopped" : `! The round could not run: ${r.error}`}`)
+  else {
+    const c = counts(r.suite.results)
+    const tokens = r.suite.results.reduce((n, x) => n + x.tokens, 0)
+    const mark = r.stopped ? "■" : c.passed === c.total ? "✓" : "✗"
+    lines.push(`[${hhmm(r.at)}] ${mark} ${c.passed}/${c.total} passing · ${formatDuration(r.suite.ms)} · ${formatTokens(tokens)} tokens${r.stopped ? " · stopped" : ""}`)
+  }
+  for (const a of r.alerts) {
+    lines.push(`    ${itemLine(a, r.at)}`)
+    const detail = a.kind === "recovered" ? "" : a.summary.split("\n").find((l) => l.trim())
+    if (detail) lines.push(`        ${detail.trim().slice(0, 200)}`)
+  }
+  const alerted = new Set(r.alerts.map((a) => a.id))
+  for (const f of r.failing.filter((f) => !alerted.has(f.id))) {
+    const why = f.alertedAt ? "already alerted" : `alert after ${r.failAfter} failing rounds`
+    lines.push(`    ✗ ${f.id} ${f.title}: failing since ${hhmm(f.since)} (${f.failures} ${f.failures === 1 ? "round" : "rounds"}, ${why})`)
+  }
+  if (r.alerts.length && !r.sent.length) lines.push("    No alert channels are set up (monitor.alerts in forgeqe.yaml), so nobody was told.")
+  const ok = r.sent.filter((s) => s.ok).map((s) => CHANNEL[s.channel])
+  if (ok.length) lines.push(`    ↑ Alert sent: ${ok.join(", ")}`)
+  for (const s of r.sent.filter((s) => !s.ok)) lines.push(`    ✗ ${CHANNEL[s.channel]} alert not sent: ${s.error}`)
+  if (r.suite) {
+    const total = suiteUsage(r.suite.results)
+    if (total.usage.input || total.usage.output) lines.push(`    Usage   ${usageLine(total.usage, total.cost)}`)
+    lines.push(`    Report  ${path.relative(cwd, r.suite.files.html) || r.suite.files.html}`)
+  }
+  if (nextAt) lines.push(`    Next round at ${hhmm(nextAt)}`)
+  return lines.join("\n")
+}
+
+/** A round as a card for the shell: the same lines as formatRound under a title. */
+export function roundCard(r: RoundResult, nextAt?: Date, cwd = process.cwd()): Card {
+  const [, ...rest] = formatRound(r, nextAt, cwd).split("\n")
+  const body = rest.map((l) => l.replace(/^ {4}/, ""))
+  if (!r.suite) return { status: r.stopped ? "stopped" : "fail", title: r.stopped ? "Monitor round stopped" : "Monitor round could not run", stats: hhmm(r.at), body: r.stopped ? body : [r.error || "", ...body] }
+  const c = counts(r.suite.results)
+  const tokens = r.suite.results.reduce((n, x) => n + x.tokens, 0)
+  return {
+    status: r.stopped ? "stopped" : c.passed === c.total ? "ok" : "fail",
+    title: `Monitor: ${c.passed}/${c.total} checks passing`,
+    stats: `${hhmm(r.at)} · ${formatDuration(r.suite.ms)} · ${formatTokens(tokens)} tokens`,
+    body,
+  }
+}
+
+/** A sample alert through every configured channel, to check the webhook and SMTP settings. */
+export async function sendTestAlert(config: AppConfig, send: typeof sendAlerts = sendAlerts): Promise<Sent[]> {
+  const at = new Date().toISOString()
+  const item: AlertItem = { kind: "failing", id: "EXAMPLE-1", title: "Sample check", since: at, failures: 1, summary: "This is a test alert from forgeqe monitor --test-alerts. No check failed." }
+  return send(config.monitor.alerts, { title: "test alert", items: [item], at, passed: 0, total: 1, test: true })
+}

package/src/monitor/state.ts

@@ -0,0 +1,105 @@
+import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs"
+import path from "node:path"
+
+import type { CaseResult, CaseStatus } from "../cases/types.ts"
+
+/** What the monitor remembers about a check between rounds. */
+export type CheckState = {
+  title: string
+  status: "passing" | "failing"
+  /** When it started passing or failing. */
+  since: string
+  /** Failing rounds in a row. */
+  failures: number
+  /** When the last alert about this failure went out. */
+  alertedAt?: string
+  summary?: string
+}
+
+export type MonitorState = { checks: Record<string, CheckState>; updatedAt?: string }
+
+export type AlertKind = "failing" | "still_failing" | "recovered"
+
+export type AlertItem = {
+  kind: AlertKind
+  id: string
+  title: string
+  /** When the check started failing (for recovered: how long it was down). */
+  since: string
+  failures: number
+  summary: string
+  result?: CaseResult
+}
+
+export type CheckOutcome = { id: string; title: string; status: CaseStatus; summary: string; result?: CaseResult }
+
+export type AlertPolicy = { failAfter: number; remindMs: number }
+
+/** The pseudo-check that fails when a round cannot run at all (a source is down, the engine will not start). */
+export const MONITOR_CHECK = { id: "forgeqe-monitor", title: "ForgeQE monitor" }
+
+/**
+ * The next state and the alerts one round's outcomes call for: a check's first alert once it has failed `failAfter`
+ * rounds in a row, a reminder every `remindMs` while it keeps failing, and a recovery once it passes after an alert.
+ * Stopped checks change nothing. `keepOthers` keeps checks this round did not report (a round that could not run).
+ */
+export function decide(prev: MonitorState, outcomes: CheckOutcome[], now: Date, policy: AlertPolicy, keepOthers = false): { state: MonitorState; alerts: AlertItem[] } {
+  const at = now.toISOString()
+  const checks: Record<string, CheckState> = keepOthers ? { ...prev.checks } : {}
+  const alerts: AlertItem[] = []
+  for (const o of outcomes) {
+    const p = prev.checks[o.id]
+    if (o.status === "stopped") {
+      if (p) checks[o.id] = p
+      continue
+    }
+    if (o.status === "failed" || o.status === "error") {
+      const ongoing = p?.status === "failing"
+      const next: CheckState = {
+        title: o.title,
+        status: "failing",
+        since: ongoing ? p.since : at,
+        failures: ongoing ? p.failures + 1 : 1,
+        ...(ongoing && p.alertedAt ? { alertedAt: p.alertedAt } : {}),
+        summary: o.summary,
+      }
+      const item = { id: o.id, title: o.title, since: next.since, failures: next.failures, summary: o.summary, result: o.result }
+      if (!next.alertedAt && next.failures >= policy.failAfter) {
+        alerts.push({ kind: "failing", ...item })
+        next.alertedAt = at
+      } else if (next.alertedAt && policy.remindMs > 0 && now.getTime() - Date.parse(next.alertedAt) >= policy.remindMs) {
+        alerts.push({ kind: "still_failing", ...item })
+        next.alertedAt = at
+      }
+      checks[o.id] = next
+    } else {
+      if (p?.status === "failing" && p.alertedAt) {
+        alerts.push({ kind: "recovered", id: o.id, title: o.title, since: p.since, failures: p.failures, summary: o.summary, result: o.result })
+      }
+      checks[o.id] = { title: o.title, status: "passing", since: p?.status === "passing" ? p.since : at, failures: 0 }
+    }
+  }
+  return { state: { checks, updatedAt: at }, alerts }
+}
+
+export function stateFile(outDir: string): string {
+  return path.join(outDir, "state.json")
+}
+
+export function readState(outDir: string): MonitorState {
+  const file = stateFile(outDir)
+  if (!existsSync(file)) return { checks: {} }
+  try {
+    const data = JSON.parse(readFileSync(file, "utf8")) as MonitorState
+    return data && typeof data.checks === "object" ? data : { checks: {} }
+  } catch {
+    return { checks: {} }
+  }
+}
+
+export function writeState(outDir: string, state: MonitorState) {
+  mkdirSync(outDir, { recursive: true })
+  const file = stateFile(outDir)
+  writeFileSync(`${file}.tmp`, `${JSON.stringify(state, null, 2)}\n`)
+  renameSync(`${file}.tmp`, file)
+}

package/src/notify/events.ts

@@ -0,0 +1,144 @@
+import os from "node:os"
+import path from "node:path"
+
+import type { AppConfig, NotifyEvent } from "../config.ts"
+import { FAILURE_LABEL } from "../cases/failure.ts"
+import type { Readiness } from "../cases/readiness.ts"
+import type { EvalReport } from "../evals/domain.ts"
+import { ciInfo, duration } from "../report/html.ts"
+import { counts } from "../suite/report.ts"
+import type { SuiteResult } from "../suite/run.ts"
+import { failureShot, sendNotice, type Link, type Notice, type NoticeItem, type SendDeps, type Sent } from "./notice.ts"
+
+/**
+ * Notifications the team gets without asking (notify: in forgeqe.yaml): each channel names the events it wants, and
+ * every event becomes the same kind of card in Teams and message in email.
+ */
+
+const MAX_CASES = 10
+const MAX_SHOTS = 5
+
+export type Notified = { channel: string; sent: Sent[] }
+
+/** Send `n` to every channel that wants `event`; the result says, per channel, what went out and what failed. */
+export async function notify(cfg: Pick<AppConfig, "notify">, event: NotifyEvent, n: Notice, deps: SendDeps = {}): Promise<Notified[]> {
+  const channels = (cfg.notify?.channels || []).filter((c) => c.on.includes(event))
+  return Promise.all(channels.map(async (c) => ({ channel: c.name, sent: await sendNotice(c, n, deps) })))
+}
+
+/** A finished suite to the channels that want it: suite_passed or suite_failed. A stopped suite is not announced. */
+export function notifySuite(cfg: Pick<AppConfig, "notify" | "export">, s: SuiteResult, deps: SendDeps = {}): Promise<Notified[]> {
+  if (s.stopped || !cfg.notify) return Promise.resolve([])
+  const n = suiteNotice(s, cfg)
+  return notify(cfg, n.status === "good" ? "suite_passed" : "suite_failed", n, deps)
+}
+
+/** "Notified qa (Teams), leads (email failed: …)": one line for a terminal or a tool result. */
+export function notifiedLine(list: Notified[]): string {
+  if (!list.length) return ""
+  const label = (c: Sent["channel"]) => (c === "teams" ? "Teams" : "email")
+  const each = list.map((n) => `${n.channel} (${n.sent.map((s) => (s.ok ? label(s.channel) : `${label(s.channel)} failed: ${s.error}`)).join(", ")})`)
+  return `${list.every((n) => n.sent.every((s) => s.ok)) ? "Notified" : "Notifications"} ${each.join("; ")}`
+}
+
+/**
+ * Where a report can be opened by whoever reads the card: under notify.report_url when the reports folder is
+ * published there, and the CI build's page when running in a pipeline. Otherwise only its path, as a note.
+ */
+export function reportLinks(cfg: Pick<AppConfig, "notify" | "export">, file?: string, env = process.env): { actions: Link[]; note?: string } {
+  const actions: Link[] = []
+  const base = cfg.notify?.reportUrl
+  if (base && file) {
+    const rel = path.relative(cfg.export.outDir, file)
+    if (rel && !rel.startsWith("..") && !path.isAbsolute(rel)) actions.push({ title: "Open report", url: `${base}/${rel.split(path.sep).map(encodeURIComponent).join("/")}` })
+  }
+  const ci = ciInfo(env).ci
+  if (ci?.url) actions.push({ title: "Open build", url: ci.url })
+  return { actions, ...(!actions.some((a) => a.title === "Open report") && file ? { note: `Report: ${file}` } : {}) }
+}
+
+function where(env = process.env): string {
+  const info = ciInfo(env)
+  return [info.ci?.name || `on ${os.hostname()}`, info.branch].filter(Boolean).join(" · ")
+}
+
+const firstLine = (s: string, max = 240) => {
+  const line = (s.split("\n").find((l) => l.trim()) || "").trim()
+  return line.length <= max ? line : `${line.slice(0, max - 1)}…`
+}
+
+/** "functional defect" → "Functional defect", for the start of a line. */
+const sentence = (s: string) => s.charAt(0).toUpperCase() + s.slice(1)
+
+export function suiteNotice(s: SuiteResult, cfg: Pick<AppConfig, "notify" | "export">, env = process.env): Notice {
+  const c = counts(s.results)
+  const notPassed = s.results.filter((r) => r.status !== "passed")
+  const testsFailed = s.results.filter((r) => r.status === "passed" && r.code && r.code.status !== "passed")
+  const ok = !s.stopped && !notPassed.length && !testsFailed.length
+  // A suite named by default ("ForgeQE suite <time>") would repeat the card's source.
+  const name = s.name.replace(/^ForgeQE suite\b/, "Suite")
+  const title = s.stopped
+    ? `${name} was stopped: ${c.passed}/${c.total} passed`
+    : ok
+      ? `${name} passed: ${c.passed}/${c.total}`
+      : `${name} failed: ${notPassed.length} of ${c.total} cases did not pass`
+  const bugs = (id: string) => s.writeBack.filter((w) => w.source === "bugs" && w.caseId === id && w.url).map((w) => ({ title: "Bug", url: w.url! }))
+  const items: NoticeItem[] = notPassed.slice(0, MAX_CASES).map((r) => ({
+    line: `${r.status === "failed" ? "✗" : r.status === "error" ? "!" : "■"} ${r.case.id} ${r.case.title}`,
+    status: r.status === "failed" ? "bad" : "warn",
+    details: [r.failure ? `${sentence(FAILURE_LABEL[r.failure.type])}: ${firstLine(r.failure.reason)}` : firstLine(r.summary)].filter(Boolean),
+    links: [...(r.case.url ? [{ title: "Test case", url: r.case.url }] : []), ...bugs(r.case.id)],
+  }))
+  if (notPassed.length > MAX_CASES) items.push({ line: `…and ${notPassed.length - MAX_CASES} more` })
+  for (const r of testsFailed.slice(0, MAX_CASES)) {
+    items.push({ line: `✗ ${r.case.id} ${r.case.title}: the case passed, but its automated test did not`, status: "bad", details: [firstLine(r.code?.error || "")].filter(Boolean) })
+  }
+  if (c.coded) items.push({ line: `Tests written: ${c.codePassed}/${c.coded} passing`, status: c.codePassed === c.coded ? "good" : "warn" })
+  const links = reportLinks(cfg, s.files.html, env)
+  return {
+    source: "ForgeQE suite",
+    title,
+    status: s.stopped ? "warn" : ok ? "good" : "bad",
+    subtitle: `${duration(s.ms)} · ${where(env)}`,
+    items,
+    actions: links.actions,
+    ...(links.note ? { note: links.note } : {}),
+    attachments: notPassed
+      .map((r) => failureShot(r.runDir, r.case.id))
+      .filter((a): a is { filename: string; path: string } => Boolean(a))
+      .slice(0, MAX_SHOTS),
+  }
+}
+
+export function readinessNotice(r: Readiness, cfg: Pick<AppConfig, "notify" | "export">, opts: { file?: string; published?: string } = {}, env = process.env): Notice {
+  const c = r.counts
+  const links = reportLinks(cfg, opts.file, env)
+  return {
+    source: "ForgeQE release readiness",
+    title: `${r.verdict} · ${r.scope}`,
+    status: r.verdict === "GO" ? "good" : r.verdict === "NO-GO" ? "bad" : "warn",
+    subtitle: `${c.stories} stories · ${c.cases} cases, ${c.executed} run, ${c.passed} passed${r.passRate === null ? "" : ` (${Math.round(r.passRate * 100)}%)`} · ${r.bugs.length} open bugs`,
+    items: [
+      ...r.blockers.map((b) => ({ line: `Blocker: ${b}`, status: "bad" as const })),
+      ...r.risks.map((b) => ({ line: `Risk: ${b}`, status: "warn" as const })),
+    ],
+    actions: [...(opts.published ? [{ title: "Open wiki page", url: opts.published }] : []), ...links.actions],
+    ...(links.note && !opts.published ? { note: links.note } : {}),
+  }
+}
+
+export function evalNotice(r: EvalReport, failed: string, cfg: Pick<AppConfig, "notify" | "export">, file?: string, env = process.env): Notice {
+  const links = reportLinks(cfg, file, env)
+  const pct = (x: { covered: number; total: number }) => (x.total ? `${Math.round((100 * x.covered) / x.total)}%` : "–")
+  return {
+    source: "ForgeQE evaluation",
+    title: `Domain knowledge below the gate: ${failed}`,
+    status: "bad",
+    subtitle: `${r.runs.length} runs · agent ${r.model || "?"} · ${where(env)}`,
+    items: (Object.keys(r.summary) as (keyof EvalReport["summary"])[]).map((v) => ({
+      line: `${v}: domain ${pct(r.summary[v].coverage.domain)} · regulation ${pct(r.summary[v].coverage.regulation)} · ${r.summary[v].invented} invented ids`,
+    })),
+    actions: links.actions,
+    ...(links.note ? { note: links.note } : {}),
+  }
+}