@monoes/monobrowse 1.0.12 → 1.0.14

package/src/report/evidence.ts

@@ -0,0 +1,379 @@
+/**
+ * Recorded evidence for a bug report (RIG-11).
+ *
+ * When a run breaches its budget, "LCP was 4.1s" is a number; the frames
+ * either side of the moment it went wrong, lined up against the console and
+ * network entries from the same instant, is a bug report. This module records
+ * the frames and assembles that timeline.
+ *
+ * Size is the hard constraint. A screencast is tens of megabytes and a report
+ * is a single self-contained HTML file, so there are three separate bounds:
+ * the recorder's ring buffer (what we hold in memory), the frame selection
+ * (what we attach), and a byte budget applied over the selection. Nothing
+ * here can produce a 200MB report.
+ *
+ * `startEvidenceRecorder` talks to Chrome; everything below it is pure and
+ * unit-tested against fixture frames.
+ */
+
+import type { CdpClient } from '../browser/cdp.js';
+import type {
+  BudgetFailure,
+  ConsoleEntry,
+  Evidence,
+  EvidenceFrame,
+  PageErrorEntry,
+  RawFrame,
+  RequestEntry,
+  TimelineEntry,
+} from './types.js';
+
+export type { RawFrame };
+
+// ---------------------------------------------------------------------------
+// Recording
+// ---------------------------------------------------------------------------
+
+export interface EvidenceRecorderOptions {
+  /** Frames held in memory before the buffer is decimated. */
+  maxFrames?: number;
+  /** Base64 bytes held in memory before the buffer is decimated. */
+  maxBytes?: number;
+  everyNthFrame?: number;
+  quality?: number;
+  maxWidth?: number;
+  maxHeight?: number;
+}
+
+export interface EvidenceRecorder {
+  frames: RawFrame[];
+  droppedFrames: number;
+  stop: () => Promise<void>;
+}
+
+const DEFAULT_MAX_BUFFER_FRAMES = 160;
+const DEFAULT_MAX_BUFFER_BYTES = 32 * 1024 * 1024;
+
+/**
+ * Halve the temporal resolution of the buffer instead of dropping its head.
+ *
+ * A plain ring buffer keeps the end of the run, which is the wrong half: a
+ * page that throws during load has its evidence at the start. Decimating
+ * keeps coverage of the whole run and merely coarsens it, which degrades the
+ * timeline gracefully however long the run turns out to be.
+ */
+function decimate(frames: RawFrame[]): { kept: RawFrame[]; dropped: number } {
+  const kept = frames.filter((_, i) => i % 2 === 0);
+  return { kept, dropped: frames.length - kept.length };
+}
+
+export async function startEvidenceRecorder(
+  client: CdpClient,
+  sessionId: string,
+  startedAtMs: number,
+  options: EvidenceRecorderOptions = {},
+): Promise<EvidenceRecorder> {
+  const maxFrames = options.maxFrames ?? DEFAULT_MAX_BUFFER_FRAMES;
+  const maxBytes = options.maxBytes ?? DEFAULT_MAX_BUFFER_BYTES;
+
+  const recorder: EvidenceRecorder = {
+    frames: [],
+    droppedFrames: 0,
+    stop: async () => {},
+  };
+  let bytes = 0;
+
+  const off = client.on('Page.screencastFrame', (params, sid) => {
+    if (sid !== sessionId) return;
+    const p = params as { data?: string; sessionId?: number };
+    // Ack first and unconditionally: Chrome stops sending frames until the
+    // previous one is acknowledged, so a failure to ack silently ends the
+    // recording.
+    if (p.sessionId !== undefined) {
+      void client
+        .send('Page.screencastFrameAck', { sessionId: p.sessionId }, sessionId)
+        .catch(() => {});
+    }
+    if (!p.data) return;
+    recorder.frames.push({
+      offsetMs: Math.max(0, Date.now() - startedAtMs),
+      data: p.data,
+      bytes: p.data.length,
+    });
+    bytes += p.data.length;
+    if (recorder.frames.length > maxFrames || bytes > maxBytes) {
+      const { kept, dropped } = decimate(recorder.frames);
+      recorder.frames = kept;
+      recorder.droppedFrames += dropped;
+      bytes = kept.reduce((sum, f) => sum + f.bytes, 0);
+    }
+  });
+
+  // Idempotent: collect() stops the recorder once the interesting part of the
+  // run is over and again in its `finally`, and Chrome errors on stopping a
+  // screencast that is already stopped.
+  let stopped = false;
+  recorder.stop = async () => {
+    if (stopped) return;
+    stopped = true;
+    off();
+    await client.send('Page.stopScreencast', {}, sessionId).catch(() => {});
+  };
+
+  try {
+    await client.send(
+      'Page.startScreencast',
+      {
+        format: 'jpeg',
+        quality: options.quality ?? 60,
+        everyNthFrame: options.everyNthFrame ?? 2,
+        maxWidth: options.maxWidth ?? 900,
+        maxHeight: options.maxHeight ?? 1600,
+      },
+      sessionId,
+    );
+  } catch (err) {
+    off();
+    throw err;
+  }
+  return recorder;
+}
+
+// ---------------------------------------------------------------------------
+// Frame selection
+// ---------------------------------------------------------------------------
+
+export interface SelectFramesOptions {
+  /** Frames to attach to the report. */
+  maxFrames?: number;
+  /** Total base64 bytes to attach. */
+  maxBytes?: number;
+  /** Half-width of the window around the focus, in ms. */
+  windowMs?: number;
+}
+
+const DEFAULT_ATTACH_FRAMES = 12;
+const DEFAULT_ATTACH_BYTES = 6 * 1024 * 1024;
+const DEFAULT_WINDOW_MS = 4000;
+
+/**
+ * Pick the frames to attach.
+ *
+ * With a focus offset, frames nearest it win — that is the moment being
+ * explained. Without one (a vitals-only breach has no instant), the run is
+ * sampled evenly so the reader still sees it load. The byte budget is applied
+ * last, evicting the least relevant frame first, so the cap is never reached
+ * by truncating the interesting end.
+ */
+export function selectFrames(
+  frames: RawFrame[],
+  focusOffsetMs: number | null,
+  options: SelectFramesOptions = {},
+): { frames: RawFrame[]; dropped: number } {
+  const maxFrames = Math.max(1, options.maxFrames ?? DEFAULT_ATTACH_FRAMES);
+  const maxBytes = options.maxBytes ?? DEFAULT_ATTACH_BYTES;
+  const windowMs = options.windowMs ?? DEFAULT_WINDOW_MS;
+  if (!frames.length) return { frames: [], dropped: 0 };
+
+  let ranked: RawFrame[];
+  if (focusOffsetMs === null) {
+    if (frames.length <= maxFrames) {
+      ranked = [...frames];
+    } else {
+      const step = (frames.length - 1) / (maxFrames - 1 || 1);
+      const picked = new Set<number>();
+      for (let i = 0; i < maxFrames; i++) picked.add(Math.round(i * step));
+      ranked = [...picked].sort((a, b) => a - b).map((i) => frames[i]);
+    }
+  } else {
+    const distance = (f: RawFrame) => Math.abs(f.offsetMs - focusOffsetMs);
+    const inWindow = frames.filter((f) => distance(f) <= windowMs);
+    // Fall back to the nearest frames overall rather than attaching none: a
+    // failure whose frames all fell outside the window still deserves context.
+    const pool = inWindow.length ? inWindow : [...frames];
+    ranked = [...pool].sort((a, b) => distance(a) - distance(b)).slice(0, maxFrames);
+  }
+
+  let dropped = frames.length - ranked.length;
+  let total = ranked.reduce((sum, f) => sum + f.bytes, 0);
+  if (total > maxBytes) {
+    const order =
+      focusOffsetMs === null
+        ? [...ranked]
+        : [...ranked].sort(
+            (a, b) => Math.abs(a.offsetMs - focusOffsetMs) - Math.abs(b.offsetMs - focusOffsetMs),
+          );
+    const keep = new Set<RawFrame>();
+    let used = 0;
+    for (const frame of order) {
+      if (used + frame.bytes > maxBytes) continue;
+      keep.add(frame);
+      used += frame.bytes;
+    }
+    dropped += ranked.length - keep.size;
+    ranked = ranked.filter((f) => keep.has(f));
+    total = used;
+  }
+
+  return { frames: ranked.sort((a, b) => a.offsetMs - b.offsetMs), dropped };
+}
+
+// ---------------------------------------------------------------------------
+// Timeline
+// ---------------------------------------------------------------------------
+
+export interface BuildEvidenceInput {
+  reason: Evidence['reason'];
+  /** `Date.now()` at the start of the run — the zero of every offset. */
+  startedAtMs: number;
+  frames: RawFrame[];
+  /** Frames the recorder already dropped to stay inside its buffer. */
+  bufferDropped?: number;
+  console: ConsoleEntry[];
+  pageErrors: PageErrorEntry[];
+  requests: RequestEntry[];
+  failures: BudgetFailure[];
+  select?: SelectFramesOptions;
+  /** Cap on non-frame timeline rows. */
+  maxEvents?: number;
+}
+
+const DEFAULT_MAX_EVENTS = 60;
+
+function truncate(text: string, max = 200): string {
+  const flat = String(text ?? '')
+    .replace(/\s+/g, ' ')
+    .trim();
+  return flat.length > max ? `${flat.slice(0, max - 1)}…` : flat;
+}
+
+/**
+ * The instant the run went wrong: the earliest hard error we have a clock
+ * reading for. Returns null when the breach has no moment attached — a slow
+ * LCP is a property of the whole load, not an event.
+ */
+export function findFocusOffset(input: BuildEvidenceInput): number | null {
+  const candidates: number[] = [];
+  for (const e of input.console) {
+    if (e.type === 'error' && Number.isFinite(e.timestamp)) {
+      candidates.push(e.timestamp - input.startedAtMs);
+    }
+  }
+  for (const e of input.pageErrors) {
+    if (Number.isFinite(e.timestamp)) candidates.push(e.timestamp - input.startedAtMs);
+  }
+  for (const r of input.requests) {
+    if (r.failed && r.startedAtMs !== undefined) candidates.push(r.startedAtMs - input.startedAtMs);
+  }
+  const usable = candidates.filter((v) => Number.isFinite(v) && v >= 0);
+  return usable.length ? Math.min(...usable) : null;
+}
+
+export function buildEvidence(input: BuildEvidenceInput): Evidence {
+  const notes: string[] = [];
+  const focusOffsetMs = findFocusOffset(input);
+  const picked = selectFrames(input.frames, focusOffsetMs, input.select);
+
+  const frames: EvidenceFrame[] = picked.frames.map((f) => ({
+    offsetMs: f.offsetMs,
+    dataUrl: `data:image/jpeg;base64,${f.data}`,
+    bytes: f.bytes,
+  }));
+
+  const events: TimelineEntry[] = [];
+  for (const [i, frame] of picked.frames.entries()) {
+    events.push({
+      offsetMs: frame.offsetMs,
+      kind: 'frame',
+      label: 'frame',
+      severity: 'info',
+      frameIndex: i,
+    });
+  }
+  for (const e of input.console) {
+    if (e.type !== 'error' && e.type !== 'warn' && e.type !== 'warning') continue;
+    events.push({
+      offsetMs: e.timestamp - input.startedAtMs,
+      kind: 'console',
+      label: `console.${e.type}`,
+      detail: truncate(e.text),
+      severity: e.type === 'error' ? 'error' : 'warning',
+    });
+  }
+  for (const e of input.pageErrors) {
+    events.push({
+      offsetMs: e.timestamp - input.startedAtMs,
+      kind: 'pageerror',
+      label: 'uncaught exception',
+      detail: truncate(e.text),
+      severity: 'error',
+    });
+  }
+  for (const r of input.requests) {
+    if (!r.failed) continue;
+    if (r.startedAtMs === undefined) continue;
+    events.push({
+      offsetMs: r.startedAtMs - input.startedAtMs,
+      kind: 'request',
+      label: `${r.status ?? r.errorText ?? 'no response'} ${r.method}`,
+      detail: truncate(r.url),
+      severity: 'error',
+    });
+  }
+  if (focusOffsetMs !== null) {
+    events.push({
+      offsetMs: focusOffsetMs,
+      kind: 'marker',
+      label: 'first failure',
+      detail: input.failures.map((f) => `${f.budget} ${f.actual}`).join(', ') || undefined,
+      severity: 'error',
+    });
+  }
+
+  // Trim the non-frame rows worst-first, so a page with 400 console warnings
+  // still shows its one uncaught exception.
+  const frameRows = events.filter((e) => e.kind === 'frame');
+  let otherRows = events.filter((e) => e.kind !== 'frame');
+  const maxEvents = input.maxEvents ?? DEFAULT_MAX_EVENTS;
+  if (otherRows.length > maxEvents) {
+    const weight = { error: 0, warning: 1, info: 2 } as const;
+    const kept = [...otherRows]
+      .sort(
+        (a, b) =>
+          weight[a.severity] - weight[b.severity] ||
+          Math.abs(a.offsetMs - (focusOffsetMs ?? 0)) - Math.abs(b.offsetMs - (focusOffsetMs ?? 0)),
+      )
+      .slice(0, maxEvents);
+    notes.push(`Timeline trimmed to the ${maxEvents} most relevant of ${otherRows.length} events.`);
+    otherRows = kept;
+  }
+
+  const timeline = [...frameRows, ...otherRows].sort(
+    (a, b) => a.offsetMs - b.offsetMs || a.kind.localeCompare(b.kind),
+  );
+
+  const droppedFrames = (input.bufferDropped ?? 0) + picked.dropped;
+  if (droppedFrames > 0) {
+    notes.push(
+      `${droppedFrames} recorded frame(s) not attached — the report keeps the ${frames.length} nearest the failure to stay inside its size budget.`,
+    );
+  }
+  if (!frames.length && input.frames.length === 0) {
+    notes.push('No screencast frames were captured for this run.');
+  }
+  if (focusOffsetMs === null && input.failures.length) {
+    notes.push(
+      'No single moment of failure could be located — the breach is a load-wide metric, so the frames sample the whole run instead.',
+    );
+  }
+
+  return {
+    reason: input.reason,
+    focusOffsetMs,
+    frames,
+    timeline,
+    droppedFrames,
+    totalBytes: frames.reduce((sum, f) => sum + f.bytes, 0),
+    notes,
+  };
+}

package/src/report/flake.ts

@@ -0,0 +1,303 @@
+/**
+ * Flake detection across repeated runs of the same URL (RIG-14).
+ *
+ * The failure mode this exists to stop: a check that fails two runs in five
+ * gets re-run until it is green and shipped as "passing". So the aggregate
+ * verdict has a third value — `flaky` — and a check that both passed and
+ * failed within the window can only produce that verdict. It is never
+ * rounded down to `pass`.
+ *
+ * The other half is honesty about what N runs can prove. `confidenceNote`
+ * states the failure rate the window actually rules out, because "5 runs
+ * passed" and "this is reliable" are very different claims.
+ *
+ * Pure: the tests feed arrays of Reports.
+ */
+
+import { budgetLabel } from './budget.js';
+import type { Budget, Report } from './types.js';
+
+export type FlakeVerdict = 'pass' | 'fail' | 'flaky';
+
+export interface FlakeCheck {
+  key: keyof Budget;
+  label: string;
+  runs: number;
+  passed: number;
+  failed: number;
+  /** Fraction of runs that failed, 0..1. */
+  flakeRate: number;
+  /** True when the check did the same thing every run. */
+  stable: boolean;
+  /** The measured value for each run, in run order. */
+  actuals: string[];
+}
+
+export interface FlakeSignal {
+  kind: 'console' | 'pageError' | 'request' | 'a11y';
+  signature: string;
+  /** Runs this signal appeared in. */
+  runs: number;
+  total: number;
+}
+
+export interface FlakeMetric {
+  key: string;
+  label: string;
+  unit: 'ms' | 'score';
+  /** Runs that reported the metric at all. */
+  measured: number;
+  total: number;
+  min: number;
+  max: number;
+  median: number;
+  spread: number;
+  /** Spread as a fraction of the median. */
+  spreadRatio: number;
+  unstable: boolean;
+}
+
+export interface FlakeReport {
+  runs: number;
+  verdict: FlakeVerdict;
+  checks: FlakeCheck[];
+  /** Errors and failed requests that did not appear in every run. */
+  signals: FlakeSignal[];
+  metrics: FlakeMetric[];
+  confidence: 'low' | 'moderate' | 'high';
+  confidenceNote: string;
+  headline: string;
+}
+
+/** Spread beyond this fraction of the median is worth flagging as unstable. */
+const UNSTABLE_SPREAD_RATIO = 0.3;
+
+const COUNT_READERS: Partial<Record<keyof Budget, (r: Report) => number>> = {
+  maxConsoleErrors: (r) => r.counts.consoleErrors,
+  maxPageErrors: (r) => r.counts.pageErrors,
+  maxFailedRequests: (r) => r.counts.failedRequests,
+  maxA11yErrors: (r) => r.counts.a11yErrors,
+};
+
+const METRIC_READERS: Partial<Record<keyof Budget, (r: Report) => number | undefined>> = {
+  lcpMs: (r) => r.vitals.lcp,
+  clsScore: (r) => r.vitals.cls,
+  inpMs: (r) => r.vitals.inp,
+  fcpMs: (r) => r.vitals.fcp,
+  ttfbMs: (r) => r.vitals.ttfb,
+};
+
+function actualFor(key: keyof Budget, report: Report): string {
+  const count = COUNT_READERS[key];
+  if (count) return String(count(report));
+  const metric = METRIC_READERS[key];
+  const raw = metric?.(report);
+  if (raw === undefined || !Number.isFinite(raw)) return 'not measured';
+  return key === 'clsScore' ? raw.toFixed(3) : `${Math.round(raw)}ms`;
+}
+
+function median(values: number[]): number {
+  const sorted = [...values].sort((a, b) => a - b);
+  const mid = Math.floor(sorted.length / 2);
+  return sorted.length % 2 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2;
+}
+
+function truncate(text: string, max = 140): string {
+  const flat = text.replace(/\s+/g, ' ').trim();
+  return flat.length > max ? `${flat.slice(0, max - 1)}…` : flat;
+}
+
+/**
+ * Normalise a message so the same error reported with a different id, line or
+ * timestamp still counts as the same signal. Without this, a flaky error
+ * carrying a request id looks like N distinct one-off errors.
+ */
+export function signalSignature(text: string): string {
+  return truncate(
+    text
+      .replace(/\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b/gi, '<uuid>')
+      .replace(/\b0x[0-9a-f]+\b/gi, '<hex>')
+      // 5 digits and up, not 3: request ids, timestamps and byte offsets are
+      // long, while an HTTP status code is part of what makes two errors
+      // different and must survive normalisation.
+      .replace(/\b\d{5,}\b/g, '<n>'),
+  );
+}
+
+function collectSignals(reports: Report[]): FlakeSignal[] {
+  const total = reports.length;
+  const seen = new Map<
+    string,
+    { kind: FlakeSignal['kind']; signature: string; runs: Set<number> }
+  >();
+
+  const add = (kind: FlakeSignal['kind'], signature: string, runIndex: number) => {
+    const id = `${kind}\n${signature}`;
+    let entry = seen.get(id);
+    if (!entry) {
+      entry = { kind, signature, runs: new Set() };
+      seen.set(id, entry);
+    }
+    entry.runs.add(runIndex);
+  };
+
+  reports.forEach((report, i) => {
+    for (const e of report.console) {
+      if (e.type === 'error') add('console', signalSignature(e.text), i);
+    }
+    for (const e of report.pageErrors) add('pageError', signalSignature(e.text), i);
+    for (const r of report.requests) {
+      if (r.failed) add('request', `${r.status ?? r.errorText ?? 'no response'} ${r.url}`, i);
+    }
+    for (const f of report.a11y) {
+      if (f.impact === 'error') add('a11y', `${f.rule} at ${f.locator}`, i);
+    }
+  });
+
+  return [...seen.values()]
+    .filter((e) => e.runs.size < total) // present in every run is a bug, not a flake
+    .map((e) => ({ kind: e.kind, signature: e.signature, runs: e.runs.size, total }))
+    .sort((a, b) => b.runs - a.runs || a.signature.localeCompare(b.signature));
+}
+
+function collectMetrics(reports: Report[]): FlakeMetric[] {
+  const defs: Array<[string, string, 'ms' | 'score', (r: Report) => number | undefined]> = [
+    ['lcp', 'LCP', 'ms', (r) => r.vitals.lcp],
+    ['fcp', 'FCP', 'ms', (r) => r.vitals.fcp],
+    ['cls', 'CLS', 'score', (r) => r.vitals.cls],
+    ['inp', 'INP', 'ms', (r) => r.vitals.inp],
+    ['ttfb', 'TTFB', 'ms', (r) => r.vitals.ttfb],
+  ];
+  const out: FlakeMetric[] = [];
+  for (const [key, label, unit, read] of defs) {
+    const values = reports
+      .map(read)
+      .filter((v): v is number => typeof v === 'number' && Number.isFinite(v));
+    if (!values.length) continue;
+    const min = Math.min(...values);
+    const max = Math.max(...values);
+    const mid = median(values);
+    const spread = max - min;
+    const spreadRatio = mid > 0 ? spread / mid : spread > 0 ? Infinity : 0;
+    out.push({
+      key,
+      label,
+      unit,
+      measured: values.length,
+      total: reports.length,
+      min,
+      max,
+      median: mid,
+      spread,
+      spreadRatio,
+      unstable: values.length > 1 && spreadRatio > UNSTABLE_SPREAD_RATIO,
+    });
+  }
+  return out;
+}
+
+/**
+ * What a window of all-passing runs actually rules out.
+ *
+ * If a check truly fails with probability p, N independent passing runs have
+ * probability (1-p)^N. Solving (1-p)^N = 0.05 gives the largest failure rate
+ * consistent with what we saw at 95% confidence — which is a far less
+ * flattering number than "N runs passed" sounds.
+ */
+export function ruledOutFailureRate(runs: number): number {
+  if (runs <= 0) return 1;
+  return 1 - 0.05 ** (1 / runs);
+}
+
+function confidenceFor(
+  runs: number,
+  verdict: FlakeVerdict,
+): { confidence: FlakeReport['confidence']; note: string } {
+  if (verdict === 'flaky') {
+    return {
+      confidence: 'high',
+      note: `Flakiness is observed, not inferred: both outcomes occurred within ${runs} runs.`,
+    };
+  }
+  const bound = Math.round(ruledOutFailureRate(runs) * 100);
+  const level = runs >= 5 ? 'high' : runs >= 3 ? 'moderate' : 'low';
+  const outcome = verdict === 'pass' ? 'passed' : 'failed';
+  return {
+    confidence: level,
+    note:
+      `${runs} run${runs === 1 ? '' : 's'}, all ${outcome} consistently. That only rules out ` +
+      `intermittent behaviour more frequent than about 1 run in ${Math.max(2, Math.round(100 / bound))} ` +
+      `(${bound}%) at 95% confidence — re-run with a larger --repeat to tighten it.`,
+  };
+}
+
+/**
+ * Aggregate N runs of the same URL.
+ *
+ * `budget` comes from the runs themselves (they all share one), and only
+ * enforced keys become checks: an unenforced budget cannot be flaky.
+ */
+export function analyzeFlake(reports: Report[]): FlakeReport {
+  if (!reports.length) throw new Error('analyzeFlake: needs at least one run');
+  const runs = reports.length;
+  const budget = reports[0].budget;
+
+  const checks: FlakeCheck[] = [];
+  for (const key of Object.keys(budget) as Array<keyof Budget>) {
+    if (budget[key] === null) continue;
+    const failed = reports.filter((r) => r.failures.some((f) => f.budget === key)).length;
+    checks.push({
+      key,
+      label: budgetLabel(key),
+      runs,
+      passed: runs - failed,
+      failed,
+      flakeRate: failed / runs,
+      stable: failed === 0 || failed === runs,
+      actuals: reports.map((r) => actualFor(key, r)),
+    });
+  }
+
+  const unstableChecks = checks.filter((c) => !c.stable);
+  const alwaysFailing = checks.filter((c) => c.failed === runs);
+  const verdict: FlakeVerdict = unstableChecks.length
+    ? 'flaky'
+    : alwaysFailing.length
+      ? 'fail'
+      : 'pass';
+
+  const signals = collectSignals(reports);
+  const metrics = collectMetrics(reports);
+  const { confidence, note } = confidenceFor(runs, verdict);
+
+  let headline: string;
+  if (verdict === 'flaky') {
+    const worst = [...unstableChecks].sort((a, b) => b.failed - a.failed)[0];
+    headline =
+      `FLAKY — ${unstableChecks.length} check${unstableChecks.length === 1 ? '' : 's'} ` +
+      `changed outcome across ${runs} runs (e.g. ${worst.label} failed ${worst.failed} of ${runs}).`;
+  } else if (verdict === 'fail') {
+    headline = `FAIL — ${alwaysFailing.length} check(s) failed in all ${runs} runs.`;
+  } else {
+    const unstableMetrics = metrics.filter((m) => m.unstable).length;
+    headline =
+      `PASS — every check held across all ${runs} runs` +
+      (unstableMetrics ? `, though ${unstableMetrics} metric(s) varied widely.` : '.');
+  }
+
+  return {
+    runs,
+    verdict,
+    checks,
+    signals,
+    metrics,
+    confidence,
+    confidenceNote: note,
+    headline,
+  };
+}
+
+/** One line for a CI log / agent transcript. */
+export function summarizeFlake(flake: FlakeReport): string {
+  return `${flake.headline} Confidence: ${flake.confidence}.`;
+}