@bugmole/cli 0.4.0

package/src/runtime/site-discovery.ts

@@ -0,0 +1,308 @@
+import YAML from "yaml";
+import type { Page } from "playwright";
+import { launcherFor } from "./browser-matrix.js";
+
+/**
+ * Discovery from nothing but a URL: a real browser opens the app and moves
+ * through it the way a first-time visitor does — following the links a person
+ * can see — and writes down every page and form it meets as runnable flows.
+ *
+ * It is read-only by design: it only follows same-origin links, never submits
+ * a form or presses a button, and steers clear of links that sign out or
+ * delete things.
+ */
+
+export type DiscoveryEvent = {
+  at: string;
+  kind: "visit" | "click" | "form" | "page" | "flows" | "done" | "skip" | "error";
+  text: string;
+  route?: string;
+};
+
+export type DiscoveredField = { label: string; type: string; required: boolean };
+export type DiscoveredForm = { name: string; fields: DiscoveredField[]; submit?: string };
+
+export type DiscoveredPage = {
+  route: string;
+  url: string;
+  title: string;
+  heading?: string;
+  /** The link labels a visitor clicks, from the home page, to reach this page. */
+  path: string[];
+  links: Array<{ label: string; route: string }>;
+  forms: DiscoveredForm[];
+};
+
+export type DiscoveredFlow = {
+  id: string;
+  name: string;
+  /** "journey" flows span several pages and are written by AI exploration. */
+  kind: "page" | "form" | "journey";
+  route: string;
+  /** Path under the spec directory, e.g. flows/discovered/sign-up.yaml. */
+  file: string;
+  content: string;
+  steps: string[];
+};
+
+export type DiscoveryResult = {
+  baseUrl: string;
+  pages: DiscoveredPage[];
+  flows: DiscoveredFlow[];
+  events: DiscoveryEvent[];
+};
+
+export type DiscoveryOptions = {
+  maxPages?: number;
+  maxDepth?: number;
+  headless?: boolean;
+  navigationTimeoutMs?: number;
+  /**
+   * Pause after each page, as a person would while reading it. Keeps the load
+   * on the app gentle. Defaults to 600ms (BUGMOLE_DISCOVERY_DWELL_MS).
+   */
+  dwellMs?: number;
+  onEvent?: (event: DiscoveryEvent, snapshot: DiscoveryResult) => void | Promise<void>;
+};
+
+const UNSAFE_LINK = /\b(log ?out|sign ?out|delete|remove|destroy|unsubscribe|deactivate|cancel (?:my )?(?:account|subscription))\b/i;
+
+export function slugify(value: string): string {
+  return value.toLowerCase().normalize("NFKD").replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 48) || "home";
+}
+
+/** The route a same-origin link leads to, or null for anything a visitor would not follow. */
+export function discoverableRoute(href: string, baseUrl: string): string | null {
+  let url: URL;
+  try {
+    url = new URL(href, baseUrl);
+  } catch {
+    return null;
+  }
+  const base = new URL(baseUrl);
+  if (url.origin !== base.origin || !/^https?:$/.test(url.protocol)) return null;
+  if (/\.(png|jpe?g|gif|svg|webp|ico|pdf|zip|css|js|json|xml|txt|mp4|webm)$/i.test(url.pathname)) return null;
+  const route = url.pathname.replace(/\/+$/, "") || "/";
+  return route;
+}
+
+function describeRoute(page: Pick<DiscoveredPage, "route" | "heading" | "title">): string {
+  if (page.route === "/") return "home page";
+  return page.heading || page.title || page.route;
+}
+
+function sampleValue(field: DiscoveredField): string {
+  const hint = `${field.type} ${field.label}`.toLowerCase();
+  if (field.type === "password" || hint.includes("password")) return "{{secret.APP_PASSWORD}}";
+  if (hint.includes("email")) return "qa@example.com";
+  if (hint.includes("phone") || field.type === "tel") return "+1 555 010 0100";
+  if (field.type === "number") return "1";
+  if (field.type === "url" || hint.includes("website")) return "https://example.com";
+  if (hint.includes("name")) return "Bugmole";
+  return "Test";
+}
+
+const FILLABLE = new Set(["text", "email", "password", "tel", "number", "url", "search", "textarea"]);
+
+/** Flows a visitor could run for each discovered page and form. */
+export function flowsFor(baseUrl: string, pages: readonly DiscoveredPage[]): DiscoveredFlow[] {
+  const flows: DiscoveredFlow[] = [];
+  const used = new Set<string>();
+  const uniqueId = (base: string) => {
+    let id = base;
+    for (let suffix = 2; used.has(id); suffix += 1) id = `${base}-${suffix}`;
+    used.add(id);
+    return id;
+  };
+  const header = (name: string) => `url: ${baseUrl}\nname: ${JSON.stringify(name)}\n---\n`;
+  const pathSteps = (page: DiscoveredPage) => page.path.map((label) => ({ tapOn: label }));
+
+  for (const page of pages) {
+    const pageName = page.route === "/" ? "Open the home page" : `Visit ${describeRoute(page)}`;
+    const landmark = page.heading ?? page.title;
+    if (landmark) {
+      const id = uniqueId(`discovered-${slugify(page.route === "/" ? "home" : describeRoute(page))}`);
+      const steps: unknown[] = ["launchApp", ...pathSteps(page), { assertVisible: landmark }];
+      flows.push({
+        id,
+        name: pageName,
+        kind: "page",
+        route: page.route,
+        file: `flows/discovered/${id}.yaml`,
+        content: header(pageName) + YAML.stringify(steps),
+        steps: steps.map(stepLabel),
+      });
+    }
+    for (const form of page.forms) {
+      const fields = form.fields.filter((field) => FILLABLE.has(field.type)).slice(0, 8);
+      if (fields.length === 0) continue;
+      const name = `Fill in ${form.name}`;
+      const id = uniqueId(`discovered-${slugify(form.name)}-form`);
+      const steps: unknown[] = [
+        "launchApp",
+        ...pathSteps(page),
+        ...fields.flatMap((field) => [{ tapOn: field.label }, { inputText: sampleValue(field) }]),
+        // Discovery never submits; the flow proves the form is ready to.
+        ...(form.submit ? [{ assertClickable: form.submit }] : []),
+      ];
+      flows.push({
+        id,
+        name,
+        kind: "form",
+        route: page.route,
+        file: `flows/discovered/${id}.yaml`,
+        content: header(name) + YAML.stringify(steps),
+        steps: steps.map(stepLabel),
+      });
+    }
+  }
+  return flows;
+}
+
+function stepLabel(step: unknown): string {
+  if (typeof step === "string") return step === "launchApp" ? "Open the app" : step;
+  const [[key, value]] = Object.entries(step as Record<string, unknown>);
+  const text = String(value);
+  switch (key) {
+    case "tapOn": return `Tap “${text}”`;
+    case "inputText": return text.startsWith("{{secret.") ? "Type the password" : `Type “${text}”`;
+    case "assertVisible": return `See “${text}”`;
+    case "assertClickable": return `“${text}” is ready`;
+    default: return key;
+  }
+}
+
+/** Reads what a visitor sees on the current page. Field values are never read. */
+const pageProbe = `() => {
+  const visible = (element) => {
+    const rect = element.getBoundingClientRect();
+    if (rect.width < 2 || rect.height < 2) return false;
+    const style = window.getComputedStyle(element);
+    return style.visibility !== "hidden" && style.display !== "none" && Number(style.opacity) !== 0;
+  };
+  const text = (element) => (element.innerText || element.textContent || "").replace(/\\s+/g, " ").trim();
+  const heading = Array.from(document.querySelectorAll("h1, h2")).find((element) => visible(element) && text(element));
+  const links = [];
+  for (const link of document.querySelectorAll("a[href]")) {
+    if (!visible(link)) continue;
+    const label = text(link) || link.getAttribute("aria-label") || "";
+    if (label && label.length <= 60) links.push({ label, href: link.href });
+  }
+  const labelOf = (field) => {
+    const own = field.labels && field.labels[0] ? text(field.labels[0]) : "";
+    return own || field.getAttribute("aria-label") || field.getAttribute("placeholder") || "";
+  };
+  const forms = [];
+  for (const form of document.querySelectorAll("form")) {
+    if (!visible(form)) continue;
+    const fields = [];
+    for (const field of form.querySelectorAll("input, textarea, select")) {
+      const type = field.tagName === "TEXTAREA" ? "textarea" : field.tagName === "SELECT" ? "select" : (field.type || "text");
+      if (!visible(field) || ["hidden", "submit", "button", "checkbox", "radio", "file"].includes(type)) continue;
+      const label = labelOf(field);
+      if (label) fields.push({ label, type, required: field.required });
+    }
+    if (!fields.length) continue;
+    const submit = form.querySelector("button[type=submit], button:not([type]), input[type=submit]");
+    const submitText = submit ? (text(submit) || submit.value || "") : "";
+    const legend = form.getAttribute("aria-label") || (form.querySelector("legend, h1, h2, h3") ? text(form.querySelector("legend, h1, h2, h3")) : "");
+    forms.push({ name: legend || submitText || "form", fields, submit: submitText || undefined });
+  }
+  return { title: document.title, heading: heading ? text(heading) : undefined, links, forms };
+}`;
+
+type Probe = { title: string; heading?: string; links: Array<{ label: string; href: string }>; forms: DiscoveredForm[] };
+
+async function readPage(page: Page): Promise<Probe> {
+  return page.evaluate((source: string) => (new Function(`return (${source})`)() as () => Probe)(), pageProbe);
+}
+
+export async function discoverSite(baseUrl: string, options: DiscoveryOptions = {}): Promise<DiscoveryResult> {
+  const start = new URL(baseUrl);
+  const root = `${start.origin}${start.pathname}`;
+  const maxPages = options.maxPages ?? 12;
+  const maxDepth = options.maxDepth ?? 2;
+  const timeout = options.navigationTimeoutMs ?? 15_000;
+  const envDwell = Number(process.env.BUGMOLE_DISCOVERY_DWELL_MS);
+  const dwellMs = options.dwellMs ?? (Number.isFinite(envDwell) && envDwell >= 0 ? envDwell : 600);
+  const result: DiscoveryResult = { baseUrl: root, pages: [], flows: [], events: [] };
+  const emit = async (kind: DiscoveryEvent["kind"], text: string, route?: string) => {
+    const event: DiscoveryEvent = { at: new Date().toISOString(), kind, text, ...(route ? { route } : {}) };
+    result.events.push(event);
+    await options.onEvent?.(event, result);
+  };
+
+  const { type, channel } = launcherFor("chromium");
+  const browser = await type.launch({ headless: options.headless ?? true, ...(channel ? { channel } : {}) });
+  try {
+    const context = await browser.newContext({ viewport: { width: 1280, height: 800 } });
+    const page = await context.newPage();
+    page.setDefaultTimeout(timeout);
+    const seen = new Set<string>();
+    // Breadth-first, so pages close to home are found first.
+    const queue: Array<{ route: string; path: string[]; parent?: string }> = [{ route: discoverableRoute(root, root) ?? "/", path: [] }];
+
+    while (queue.length > 0 && result.pages.length < maxPages) {
+      const next = queue.shift()!;
+      if (seen.has(next.route)) continue;
+      seen.add(next.route);
+      try {
+        if (next.path.length === 0) {
+          await emit("visit", "Visiting home page", next.route);
+          await page.goto(root, { waitUntil: "load" });
+        } else {
+          // Reach the page the way a visitor would: from its parent, by clicking.
+          const parentUrl = new URL(next.parent ?? "/", root).toString();
+          if (discoverableRoute(page.url(), root) !== next.parent) await page.goto(parentUrl, { waitUntil: "load" });
+          const label = next.path[next.path.length - 1];
+          await emit("click", `Clicking “${label}”`, next.route);
+          await page.getByRole("link", { name: label, exact: true }).first().click();
+          await page.waitForLoadState("load").catch(() => undefined);
+          await page.waitForTimeout(250);
+        }
+        const route = discoverableRoute(page.url(), root) ?? next.route;
+        if (route !== next.route && seen.has(route)) continue;
+        seen.add(route);
+        const probe = await readPage(page);
+        const links: DiscoveredPage["links"] = [];
+        for (const link of probe.links) {
+          const target = discoverableRoute(link.href, root);
+          if (!target || target === route || links.some((existing) => existing.route === target)) continue;
+          if (UNSAFE_LINK.test(link.label)) {
+            await emit("skip", `Skipping “${link.label}” (it could change data)`, target);
+            continue;
+          }
+          links.push({ label: link.label, route: target });
+        }
+        const discovered: DiscoveredPage = {
+          route,
+          url: page.url(),
+          title: probe.title,
+          ...(probe.heading ? { heading: probe.heading } : {}),
+          path: next.path,
+          links,
+          forms: probe.forms,
+        };
+        result.pages.push(discovered);
+        await emit("page", `Found ${describeRoute(discovered)} (${links.length} link${links.length === 1 ? "" : "s"})`, route);
+        for (const form of probe.forms) {
+          await emit("form", `Found a form: ${form.name} (${form.fields.map((field) => field.label).join(", ")})`, route);
+        }
+        if (dwellMs > 0) await page.waitForTimeout(dwellMs);
+        if (next.path.length < maxDepth) {
+          for (const link of links) {
+            if (!seen.has(link.route)) queue.push({ route: link.route, path: [...next.path, link.label], parent: route });
+          }
+        }
+      } catch (error) {
+        await emit("error", `Could not open ${next.route}: ${error instanceof Error ? error.message.split("\n")[0] : String(error)}`, next.route);
+      }
+    }
+    result.flows = flowsFor(root, result.pages);
+    await emit("flows", `Discovered ${result.flows.length} flow${result.flows.length === 1 ? "" : "s"} across ${result.pages.length} page${result.pages.length === 1 ? "" : "s"}`);
+    await emit("done", "Exploration complete");
+    return result;
+  } finally {
+    await browser.close();
+  }
+}

package/src/runtime/target-runner.ts

@@ -0,0 +1,144 @@
+import { cloudDeviceLabel } from "./device-clouds/types.js";
+import type { DriverResult, DriverStepResult } from "./driver.js";
+import type { BrowserName, RunTarget } from "./browser-matrix.js";
+
+export type TargetStatus = "queued" | "running" | "passed" | "failed" | "blocked";
+
+/** One browser/device's outcome within a run, as reported to the registry and dashboard. */
+export type TargetResult = {
+  targetId: string;
+  browser: BrowserName;
+  device?: string;
+  /** Set when the target ran on a device cloud. */
+  cloudProvider?: string;
+  status: TargetStatus;
+  startedAt?: string;
+  completedAt?: string;
+  durationMs?: number;
+  stepsPassed: number;
+  stepsFailed: number;
+  failureCode?: string;
+  message?: string;
+  recoveryAttempts?: number;
+  artifactKeys: string[];
+  finalScreenshotKey?: string;
+  logKey?: string;
+  traceKey?: string;
+  videoKey?: string;
+  analysisKey?: string;
+  /** Each step as it ran, for the run's trace timeline. Typed text is masked. */
+  steps?: Array<{ index: number; status: "passed" | "failed" | "blocked"; label?: string; durationMs?: number; message?: string }>;
+  /** A short form of analysis.json, so the dashboard can show it without a fetch. */
+  analysis?: { title: string; category: "test" | "app" | "environment"; source: "rules" | "ai"; fixSummary?: string };
+};
+
+export function queuedTargetResult(target: RunTarget): TargetResult {
+  return {
+    targetId: target.id,
+    browser: target.browser,
+    ...(target.device ? { device: target.device } : {}),
+    // A cloud device shows as the device people picked, with where it ran.
+    ...(target.cloud ? { device: cloudDeviceLabel(target.cloud), cloudProvider: target.cloud.provider } : {}),
+    status: "queued",
+    stepsPassed: 0,
+    stepsFailed: 0,
+    artifactKeys: [],
+  };
+}
+
+export function stepCounts(result: DriverResult): { stepsPassed: number; stepsFailed: number } {
+  const steps = result.steps ?? [];
+  return {
+    stepsPassed: steps.filter((step) => step.status === "passed").length,
+    stepsFailed: steps.filter((step) => step.status === "failed").length,
+  };
+}
+
+/**
+ * Runs `work` for every item with at most `limit` in flight at once, keeping
+ * results in input order. A throwing item does not stop the others; callers
+ * turn failures into results themselves.
+ */
+export async function runPool<T, R>(items: readonly T[], limit: number, work: (item: T, index: number) => Promise<R>): Promise<R[]> {
+  const results = new Array<R>(items.length);
+  let next = 0;
+  const lanes = Math.max(1, Math.min(limit, items.length));
+  await Promise.all(
+    Array.from({ length: lanes }, async () => {
+      for (;;) {
+        const index = next;
+        next += 1;
+        if (index >= items.length) return;
+        results[index] = await work(items[index], index);
+      }
+    }),
+  );
+  return results;
+}
+
+export function resolveTargetConcurrency(
+  configured: unknown,
+  targetCount: number,
+  env: Record<string, string | undefined> = process.env,
+): number {
+  const fromEnv = Number(env.BUGMOLE_TARGET_CONCURRENCY);
+  if (Number.isInteger(fromEnv) && fromEnv > 0) return fromEnv;
+  const fromConfig = Number(configured);
+  if (Number.isInteger(fromConfig) && fromConfig > 0) return fromConfig;
+  return Math.max(1, Math.min(targetCount, 4));
+}
+
+const stepRank: Record<DriverStepResult["status"], number> = { passed: 0, blocked: 1, failed: 2 };
+
+/**
+ * Folds per-target driver results into the one verdict the rest of the
+ * pipeline (step correspondence, run status) already understands.
+ *
+ * - Any failed target fails the run.
+ * - Targets that could not run at all (a browser not installed, an
+ *   unsupported pairing) are reported but do not fail a run whose other
+ *   targets passed; if none could run, the run is blocked.
+ * - Per-step statuses take the worst outcome across targets that ran, and
+ *   are only produced when those targets agree on the step count, the same
+ *   rule the executor applies to a single driver.
+ */
+export function aggregateTargetResults(results: ReadonlyArray<{ target: RunTarget; result: DriverResult }>): DriverResult {
+  const ran = results.filter(({ result }) => result.status !== "blocked");
+  const failed = results.filter(({ result }) => result.status === "failed");
+  const describe = ({ target, result }: { target: RunTarget; result: DriverResult }) => `${target.id}: ${result.message}`;
+
+  let status: DriverResult["status"];
+  let message: string;
+  if (failed.length > 0) {
+    status = "failed";
+    message = `${failed.length} of ${results.length} target(s) failed. ${failed.map(describe).join(" | ")}`;
+  } else if (ran.length === 0) {
+    status = "blocked";
+    message = `No target could run. ${results.map(describe).join(" | ")}`;
+  } else {
+    status = "passed";
+    const skipped = results.length - ran.length;
+    message = `${ran.length} target(s) passed${skipped ? `; ${skipped} could not run` : ""}.`;
+  }
+
+  let steps: DriverStepResult[] | undefined;
+  const stepLists = ran.map(({ result }) => result.steps);
+  if (stepLists.length > 0 && stepLists.every((list) => list && list.length === stepLists[0]!.length)) {
+    steps = stepLists[0]!.map((first, index) =>
+      stepLists.reduce<DriverStepResult>((worst, list) => {
+        const candidate = list![index];
+        return stepRank[candidate.status] > stepRank[worst.status] ? candidate : worst;
+      }, first),
+    );
+  }
+
+  const firstProblem = failed[0] ?? (ran.length === 0 ? results[0] : undefined);
+  return {
+    status,
+    message,
+    artifacts: [],
+    ...(firstProblem?.result.failureCode ? { failureCode: firstProblem.result.failureCode } : {}),
+    ...(failed.length > 0 ? { recoverable: failed.every(({ result }) => result.recoverable) } : {}),
+    ...(steps ? { steps } : {}),
+  };
+}

package/src/runtime/test-case-verdicts.test.ts

@@ -0,0 +1,94 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { verdictsFromRun } from "./test-case-verdicts.js";
+
+const cases = [
+  { caseId: "TC-home", nodeId: "screen:/" },
+  { caseId: "TC-login", nodeId: "screen:/login" },
+];
+
+test("a node the run passed judges its case passed", () => {
+  const [home] = verdictsFromRun([cases[0]], [{ nodeId: "screen:/", outcome: "PASS", status: "success" }]);
+  assert.equal(home.status, "passed");
+  assert.match(home.detail, /screen:\//);
+});
+
+test("a node the run failed judges its case failed, carrying the reason", () => {
+  const [home] = verdictsFromRun([cases[0]], [{
+    nodeId: "screen:/",
+    outcome: "FAIL",
+    status: "failed",
+    reason: { code: "ASSERT_VISIBLE", message: "h1 was not found" },
+  }]);
+  assert.equal(home.status, "failed");
+  assert.match(home.detail, /h1 was not found/);
+  assert.match(home.detail, /ASSERT_VISIBLE/);
+});
+
+test("a case the run never reached is blocked, never passed", () => {
+  // The whole point: silence is not evidence. Marking this passed would let
+  // a run that touched one node claim coverage over a whole flow.
+  const verdicts = verdictsFromRun(cases, [{ nodeId: "screen:/", outcome: "PASS", status: "success" }]);
+  const login = verdicts.find((verdict) => verdict.caseId === "TC-login")!;
+  assert.equal(login.status, "blocked");
+  assert.match(login.detail, /never exercised/);
+});
+
+test("every case gets a verdict, so the flow cannot stay queued forever", () => {
+  // A case left unjudged is indistinguishable from one never run, which is
+  // what made the pipeline re-queue the same flow indefinitely.
+  const verdicts = verdictsFromRun(cases, []);
+  assert.equal(verdicts.length, cases.length);
+  assert.ok(verdicts.every((verdict) => verdict.status !== "passed"));
+});
+
+test("a blocked node reports why it could not be checked", () => {
+  const [home] = verdictsFromRun([cases[0]], [{
+    nodeId: "screen:/",
+    status: "blocked",
+    reason: { code: "NO_SESSION", message: "not authenticated" },
+  }]);
+  assert.equal(home.status, "blocked");
+  assert.match(home.detail, /not authenticated/);
+});
+
+test("an UNKNOWN outcome is blocked rather than quietly passed", () => {
+  const [home] = verdictsFromRun([cases[0]], [{ nodeId: "screen:/", outcome: "UNKNOWN" }]);
+  assert.equal(home.status, "blocked");
+});
+
+test("a failure anywhere at the node outranks a pass elsewhere at it", () => {
+  const [home] = verdictsFromRun([cases[0]], [
+    { nodeId: "screen:/", outcome: "PASS", status: "success" },
+    { nodeId: "screen:/", outcome: "FAIL", status: "failed", reason: { message: "second action failed" } },
+  ]);
+  assert.equal(home.status, "failed");
+  assert.match(home.detail, /second action failed/);
+});
+
+test("an assumption already flagged blocked keeps its own reason", () => {
+  const [only] = verdictsFromRun(
+    [{ caseId: "TC-x", nodeId: "screen:/", blockedReason: "needs a seeded account" }],
+    [{ nodeId: "screen:/", outcome: "PASS", status: "success" }],
+  );
+  assert.equal(only.status, "blocked");
+  assert.equal(only.detail, "needs a seeded account");
+});
+
+test("a crashed run names the real error instead of a generic 'recorded nothing'", () => {
+  // A run that threw reached no node at all. Without the reason every case
+  // would read "the run recorded nothing here", hiding why.
+  const verdicts = verdictsFromRun(cases, [], "The run failed 3 times. Last error: browser launch failed");
+  assert.equal(verdicts.length, 2);
+  assert.ok(verdicts.every((verdict) => verdict.status === "blocked"));
+  assert.match(verdicts[0].detail, /browser launch failed/);
+});
+
+test("the crash reason does not override what the run actually observed", () => {
+  const [home] = verdictsFromRun(
+    [cases[0]],
+    [{ nodeId: "screen:/", outcome: "PASS", status: "success" }],
+    "some run-level failure",
+  );
+  assert.equal(home.status, "passed");
+});

package/src/runtime/test-case-verdicts.ts

@@ -0,0 +1,120 @@
+/**
+ * Turning a run into test case verdicts.
+ *
+ * The pipeline's run stage is satisfied only when every case in a flow has a
+ * recorded verdict, and the run is what is supposed to produce them. Without
+ * this the two halves never met: runs recorded step counts and evidence, the
+ * cases stayed "planned", and the pipeline re-queued the same flow forever
+ * waiting for something no run ever wrote.
+ *
+ * Cases are pinned to a nodeId and the executor reports per-node outcomes, so
+ * the join is the node: a case is judged by what happened at the step it is
+ * about.
+ */
+
+export type NodeOutcome = {
+  nodeId: string;
+  /** The driver's verdict for this node. */
+  outcome?: "PASS" | "FAIL" | "UNKNOWN";
+  status?: "success" | "blocked" | "failed";
+  reason?: { code?: string; message?: string };
+};
+
+export type JudgeableCase = {
+  caseId: string;
+  nodeId: string;
+  title?: string;
+  blockedReason?: string;
+};
+
+export type CaseVerdict = {
+  caseId: string;
+  status: "passed" | "failed" | "blocked" | "needs_setup";
+  detail: string;
+};
+
+function describe(node: NodeOutcome): string {
+  const message = node.reason?.message?.trim();
+  const code = node.reason?.code?.trim();
+  if (message) return code ? `${message} (${code})` : message;
+  if (code) return code;
+  return "no reason recorded";
+}
+
+/**
+ * Judge one flow's cases from what the run observed at each node.
+ *
+ * Every case gets a verdict, including the ones the run never reached —
+ * those are "blocked" with that stated as the reason. That completeness is
+ * the point: a case left unjudged is indistinguishable from one nobody has
+ * run yet, so it would keep the flow queued forever. "Blocked with a reason"
+ * is a real answer and lets the pipeline move on and surface it.
+ *
+ * A case is never marked passed on absence of evidence. If the run said
+ * nothing about its node, that is not a pass.
+ */
+export function verdictsFromRun(
+  cases: JudgeableCase[],
+  nodes: NodeOutcome[],
+  /**
+   * Why the run reached nothing, when it failed outright rather than simply
+   * not visiting a node. Without it a crashed run reads as "recorded nothing
+   * here", which hides the actual cause behind a generic message.
+   */
+  unreachedReason?: string,
+): CaseVerdict[] {
+  const byNode = new Map<string, NodeOutcome[]>();
+  for (const node of nodes) {
+    if (!node?.nodeId) continue;
+    const list = byNode.get(node.nodeId) ?? [];
+    list.push(node);
+    byNode.set(node.nodeId, list);
+  }
+
+  return cases.map((testCase) => {
+    // An assumption the author already flagged as blocked stays blocked: the
+    // run cannot un-block something that was never executable.
+    if (testCase.blockedReason) {
+      return {
+        caseId: testCase.caseId,
+        status: "blocked" as const,
+        detail: testCase.blockedReason,
+      };
+    }
+
+    const observed = byNode.get(testCase.nodeId) ?? [];
+    if (observed.length === 0) {
+      return {
+        caseId: testCase.caseId,
+        status: "blocked" as const,
+        detail: unreachedReason
+          ?? `The run recorded nothing at ${testCase.nodeId}, so this case was never exercised.`,
+      };
+    }
+
+    const failed = observed.filter((node) => node.outcome === "FAIL" || node.status === "failed");
+    if (failed.length > 0) {
+      return {
+        caseId: testCase.caseId,
+        status: "failed" as const,
+        detail: `The run failed at ${testCase.nodeId}: ${describe(failed[0])}`,
+      };
+    }
+
+    const blocked = observed.filter((node) => node.status === "blocked" || node.outcome === "UNKNOWN");
+    if (blocked.length > 0) {
+      return {
+        caseId: testCase.caseId,
+        status: "blocked" as const,
+        detail: `The run could not check ${testCase.nodeId}: ${describe(blocked[0])}`,
+      };
+    }
+
+    return {
+      caseId: testCase.caseId,
+      status: "passed" as const,
+      detail: `The run passed every step at ${testCase.nodeId}`
+        + `${observed.length > 1 ? ` (${observed.length} actions)` : ""}.`,
+    };
+  });
+}