automatica11y 0.0.0-stage → 0.3.0

package/src/report/parts.js

@@ -0,0 +1,334 @@
+import { cap, num, plural } from "../text.js";
+
+/** Markdown helpers. */
+export const cell = (text) => String(text ?? "").replace(/\|/g, "\\|").replace(/\n/g, " ");
+export const code = (text) => `\`${String(text).replace(/`/g, "'")}\``;
+/** Keep a reason from ending in two periods. */
+export const sentence = (text) => String(text).replace(/\.+$/, "");
+/** Escape angle brackets so rule text like <input> doesn't turn into HTML. */
+export const esc = (text) => String(text ?? "").replace(/</g, "\\<");
+export const ENGINE_NAMES = { axe: "axe-core", ibm: "IBM Equal Access" };
+export const TIER_NAMES = { rules: "Rules", interactions: "Interactions", vsr: "Virtual screen reader" };
+
+export function toolLines(tools) {
+  return Object.entries(tools)
+    .filter(([, version]) => version)
+    .map(([name, version]) => `- ${name}: ${version}`);
+}
+
+/** The text for one coverage-matrix cell. */
+export function engineCell(summary, engine) {
+  const s = summary?.engines?.[engine];
+  if (!s) return "-";
+  if (s.status !== "ran") return s.status;
+  return `ran, ${plural(s.violations, "violation")}`;
+}
+
+export function tierCell(target, tier) {
+  const counts = target.summary?.interactions;
+  const vsr = target.summary?.vsr;
+  if (tier === "vsr" && vsr) return `ran (simulated), ${vsr.flagged ? `${num(vsr.flagged)} flagged` : "none flagged"}`;
+  if (tier === "interactions" && counts) {
+    const parts = [counts.fail && plural(counts.fail, "failed", "failed"), counts.error && plural(counts.error, "error"), counts.pass && `${num(counts.pass)} passed`, counts.notApplicable && `${num(counts.notApplicable)} not applicable`].filter(Boolean);
+    return `ran, ${parts.join(", ")}`;
+  }
+  const configs = Object.values(target.archetypes).flatMap((a) => a.configs);
+  const status = configs.map((c) => c.tiers[tier]?.status).find(Boolean);
+  return status ?? "-";
+}
+
+export function coverageMatrix(plan, results) {
+  const engines = plan.options.engines;
+  const tiers = plan.options.tiers.filter((t) => t !== "rules");
+  const rulesRequested = plan.options.tiers.includes("rules");
+  const head = ["Target", ...(rulesRequested ? engines.map((e) => ENGINE_NAMES[e]) : []), ...tiers.map((t) => TIER_NAMES[t])];
+  const rows = results.targets.map((target) => {
+    if (target.status === "failed") return [cell(target.id), ...head.slice(1).map(() => "failed")];
+    return [
+      cell(target.id),
+      ...(rulesRequested ? engines.map((e) => engineCell(target.summary, e)) : []),
+      ...tiers.map((t) => tierCell(target, t)),
+    ];
+  });
+  return [`| ${head.join(" | ")} |`, `| ${head.map(() => "---").join(" | ")} |`, ...rows.map((r) => `| ${r.join(" | ")} |`)].join("\n");
+}
+
+export function findingBlock(finding, { showImpact, showToolkit }) {
+  const meta = [
+    showImpact && finding.impact ? `impact: ${finding.impact}` : null,
+    showToolkit && finding.toolkitLevel != null ? `IBM Toolkit level ${finding.toolkitLevel}` : null,
+    finding.kind ? `kind: ${finding.kind}` : null,
+    finding.wcag.length ? `WCAG ${finding.wcag.join(", ")}` : null,
+    plural(finding.nodeCount, "element"),
+  ].filter(Boolean);
+  const lines = [`- ${code(finding.ruleId)} (${meta.join("; ")}). ${esc(finding.help)} [Rule help](${finding.helpUrl})`];
+  for (const node of finding.nodes) lines.push(`  - ${code(node.selector)} ${node.html ? code(node.html) : ""}`.trimEnd());
+  if (finding.nodeCount > finding.nodes.length) lines.push(`  - and ${num(finding.nodeCount - finding.nodes.length)} more`);
+  return lines.join("\n");
+}
+
+export function engineSection(engine, result, nested = false) {
+  const title = `${nested ? "#####" : "####"} ${ENGINE_NAMES[engine] ?? engine}${result.version ? ` ${result.version}` : ""}.`;
+  if (result.status === "not-testable") return `${title}\n\nNot testable: ${sentence(result.reason)}. Treat this as a coverage gap, not a pass.\n`;
+  if (result.status !== "ran") return `${title}\n\nThis engine didn't run: ${sentence(result.reason ?? result.status)}. Treat this as a coverage gap, not a pass.\n`;
+  const showImpact = engine === "axe";
+  const showToolkit = engine === "ibm";
+  const cfg = result.config ? `Configuration: ${code(JSON.stringify(result.config))}.` : "";
+  const out = [title, "", cfg, ""];
+  out.push("**Violations.**", "");
+  if (result.violations.length === 0) out.push(`No automated violations found by ${ENGINE_NAMES[engine]}.`);
+  else out.push(...result.violations.map((f) => findingBlock(f, { showImpact, showToolkit })));
+  out.push("", "**Needs review.**", "");
+  if (result.incomplete.length === 0) out.push("Nothing flagged for review.");
+  else out.push(...result.incomplete.map((f) => findingBlock(f, { showImpact, showToolkit })));
+  out.push("", "**Passes.**", "", `${cap(plural(result.passesCount, "rule"))} passed. A pass means the rule found nothing to flag. It doesn't show the page meets those criteria.`);
+  for (const note of result.notes ?? []) out.push("", `Note: ${note}`);
+  return out.join("\n");
+}
+
+
+/** Group one engine's findings across every story, so a long Storybook reads as rules, not as hundreds of repeats. */
+export function aggregate(target, engine, listName) {
+  const byRule = new Map();
+  let version = null;
+  let ranStories = 0;
+  for (const [key, archetype] of Object.entries(target.archetypes)) {
+    const result = archetype.configs[0]?.tiers.rules?.engines?.[engine];
+    if (!result || result.status !== "ran") continue;
+    version ??= result.version;
+    ranStories += 1;
+    for (const finding of result[listName]) {
+      const id = `${finding.ruleId}|${finding.kind ?? ""}`;
+      const entry = byRule.get(id) ?? { finding, stories: [] };
+      entry.stories.push(key.replace(/^story:/, ""));
+      byRule.set(id, entry);
+    }
+  }
+  return { rows: [...byRule.values()].sort((a, b) => b.stories.length - a.stories.length || a.finding.ruleId.localeCompare(b.finding.ruleId)), version, ranStories };
+}
+
+export function ruleTable(rows, engine) {
+  if (rows.length === 0) return ["None."];
+  const head = engine === "axe" ? "Impact" : "Toolkit level";
+  const lines = [`| Rule | ${head} | WCAG | Stories | Examples |`, "| --- | --- | --- | --- | --- |"];
+  for (const { finding, stories } of rows) {
+    const level = engine === "axe" ? finding.impact ?? "-" : finding.toolkitLevel ?? "-";
+    const rule = finding.kind ? `${finding.ruleId} (${finding.kind})` : finding.ruleId;
+    lines.push(`| ${cell(code(rule))} | ${level} | ${cell(finding.wcag.join(", ") || "-")} | ${stories.length} | ${cell(stories.slice(0, 3).join(", "))}${stories.length > 3 ? ", and more" : ""} |`);
+  }
+  return lines;
+}
+
+export function storybookSection(planTarget, target) {
+  const sb = target.storybook;
+  const lines = [`### ${planTarget.label}.`, "", `Storybook ${code(planTarget.input)}, component evidence. Each story ran as its own unit, and rules were scoped to the story's root element.`, ""];
+  const filtered = sb.filtered ?? sb.matched !== sb.total;
+  lines.push(
+    `The index lists ${plural(sb.total, "story", "stories")}. ${filtered ? `${cap(num(sb.matched))} matched the archetype filter. ` : ""}${cap(num(sb.audited))} ${sb.audited === 1 ? "was" : "were"} audited, with a cap of ${num(sb.maxStories)}.`,
+    "",
+  );
+  for (const warning of target.warnings) lines.push(`Warning: ${warning}`, "");
+  const matches = Object.entries(sb.archetypeMatches);
+  if (matches.length) {
+    lines.push("**Archetype matches.** Matches come from story titles, names, and tags, so check them.", "");
+    for (const [archetype, ids] of matches) lines.push(`- ${archetype}: ${ids.slice(0, 8).map(code).join(", ")}${ids.length > 8 ? `, and ${num(ids.length - 8)} more` : ""}`);
+    lines.push("");
+  }
+  if (sb.failedStories.length) {
+    lines.push("**Stories that didn't render.** Each one is a gap in coverage, not a pass.", "");
+    for (const failure of sb.failedStories) lines.push(`- ${code(failure.id)}: ${sentence(failure.reason)}.`);
+    lines.push("");
+  }
+  for (const engine of Object.keys(target.summary.engines)) {
+    const violations = aggregate(target, engine, "violations");
+    const review = aggregate(target, engine, "incomplete");
+    lines.push(`#### ${ENGINE_NAMES[engine] ?? engine}${violations.version ? ` ${violations.version}` : ""}.`, "");
+    if (violations.ranStories === 0) {
+      lines.push("This engine didn't run on any story. Treat that as a coverage gap, not a pass.", "");
+      continue;
+    }
+    lines.push("**Violations by rule.**", "");
+    if (violations.rows.length === 0) lines.push(`No automated violations found by ${ENGINE_NAMES[engine]} across ${plural(violations.ranStories, "story", "stories")}.`);
+    else lines.push(...ruleTable(violations.rows, engine));
+    lines.push("", "**Needs review by rule.**", "", ...ruleTable(review.rows, engine));
+    lines.push("", "Element-level detail for every story is in results.json.", "");
+  }
+  const walks = Object.entries(target.archetypes).map(([key, a]) => ({ id: key.replace(/^story:/, ""), vsr: a.configs[0]?.tiers.vsr })).filter((w) => w.vsr?.status === "ran");
+  if (walks.length) {
+    /** @type {Map<string, { flag: any, stories: string[] }>} */
+    const byFlag = new Map();
+    for (const { id, vsr } of walks) for (const flag of vsr.flags) {
+      const key = `${flag.type}|${flag.phrase}`;
+      const entry = byFlag.get(key) ?? { flag, stories: [] };
+      entry.stories.push(id);
+      byFlag.set(key, entry);
+    }
+    lines.push("#### Virtual screen reader (simulated).", "", `Simulated output from @guidepup/virtual-screen-reader${walks[0].vsr.version ? ` ${walks[0].vsr.version}` : ""}. It isn't a real screen reader. ${cap(num(walks.length))} ${walks.length === 1 ? "story was" : "stories were"} walked. The flags only mark a phrase that is a bare role or a generic role, for a person to check. Full announcement logs are in results.json.`, "");
+    if (byFlag.size === 0) lines.push("No phrases flagged.", "");
+    else {
+      lines.push("| Phrase | Why it's flagged | Stories | Examples |", "| --- | --- | --- | --- |");
+      for (const { flag, stories } of [...byFlag.values()].sort((a, b) => b.stories.length - a.stories.length)) {
+        lines.push(`| ${cell(code(flag.phrase))} | ${FLAG_TEXT[flag.type] ?? flag.type} | ${stories.length} | ${cell(stories.slice(0, 3).join(", "))}${stories.length > 3 ? ", and more" : ""} |`);
+      }
+      lines.push("");
+    }
+  }
+  return lines.join("\n");
+}
+
+export const FLAG_TEXT = {
+  "unnamed-control": "announced as only its role, with no name",
+  "generic-role": "announced with a generic role",
+};
+
+export function vsrSection(result, nested) {
+  const lines = [`${nested ? "#####" : "####"} Virtual screen reader (simulated).`, ""];
+  lines.push(`Simulated output from @guidepup/virtual-screen-reader${result.version ? ` ${result.version}` : ""}. It isn't a real screen reader, and real ones announce things differently. The log is data. The flags only mark a phrase that is a bare role or a generic role, for a person to check.`, "");
+  for (const note of result.notes ?? []) lines.push(`Note: ${note}`, "");
+  for (const entry of result.notTestable ?? []) lines.push(`Not testable: ${entry}.`, "");
+  lines.push("**Flagged phrases.**", "");
+  if (result.flags.length === 0) lines.push("None.");
+  else for (const flag of result.flags) lines.push(`- ${code(flag.phrase)} at position ${flag.index + 1}: ${FLAG_TEXT[flag.type] ?? flag.type}.`);
+  for (const entry of result.log) {
+    const shown = entry.announcements.slice(0, 40);
+    lines.push("", `**Announcement log${entry.state ? ` (${entry.state} state)` : ""}.**`, "", "```text", ...shown, ...(entry.announcements.length > shown.length ? [`... and ${num(entry.announcements.length - shown.length)} more in results.json`] : []), "```");
+  }
+  return lines.join("\n");
+}
+
+export function interactionsSection(result, nested) {
+  const lines = [`${nested ? "#####" : "####"} Interactions.`, "", "Each check ran on a fresh page, using only the trigger and root hooks and ARIA roles. A check that couldn't finish is an error, which counts as a gap and never as a pass.", ""];
+  lines.push("| Check | Result | WCAG | Detail |", "| --- | --- | --- | --- |");
+  for (const check of result.checks) {
+    lines.push(`| ${code(check.name)} | ${check.result} | ${cell((check.criteria ?? []).join(", ") || "-")} | ${cell(check.detail)}${check.method ? cell(` (method: ${check.method})`) : ""} |`);
+  }
+  return lines.join("\n");
+}
+
+/** " (open state, library accessibility on)" for a config. */
+export function configLabel(config) {
+  const parts = [config.state ? `${config.state} state` : null, config.libA11y && config.libA11y !== "n/a" ? `library accessibility ${config.libA11y}` : null].filter(Boolean);
+  return parts.length ? ` (${parts.join(", ")})` : "";
+}
+
+export function archetypeTable(target) {
+  const rows = Object.entries(target.archetypes).map(([name, archetype]) => {
+    if (archetype.status === "gap") return `| ${name} | gap | - | ${cell(sentence(archetype.reason ?? "No fixture."))} |`;
+    const states = [...new Set(archetype.configs.map((c) => c.state).filter(Boolean))].join(", ") || "-";
+    const libs = [...new Set(archetype.configs.map((c) => c.libA11y).filter((l) => l && l !== "n/a"))];
+    return `| ${name} | ran | ${states} | ${libs.length ? `library accessibility ${libs.join(" and ")}` : ""} |`;
+  });
+  return ["| Archetype | Status | States | Note |", "| --- | --- | --- | --- |", ...rows];
+}
+
+export function targetSection(planTarget, target) {
+  if (target.status === "ran" && target.storybook) return storybookSection(planTarget, target);
+  const lines = [`### ${planTarget.label}.`, ""];
+  lines.push(`Target ${code(planTarget.input)}, ${planTarget.kind ?? "unclassified"}${planTarget.evidenceLevel ? `, ${planTarget.evidenceLevel} evidence` : ""}.`, "");
+  if (target.status !== "ran" && target.status !== "failed") {
+    lines.push(`This target is ${target.status}: ${sentence(target.reason)}. That's a gap in coverage. It isn't a pass.`, "");
+    return lines.join("\n");
+  }
+  if (target.npm) {
+    const n = target.npm;
+    lines.push(`Installed ${code(`${n.name}@${n.version}`)} on its own, as ${n.flavor === "react" ? `React${n.react ? ` (react ${n.react})` : ""}` : `web components (${n.tags.length ? n.tags.slice(0, 6).join(", ") : "no tags found"})`}.`, "");
+  }
+  for (const warning of target.warnings) lines.push(`Warning: ${warning}`, "");
+  if (target.status === "failed") {
+    lines.push(`This target failed: ${sentence(target.reason)}. A failed target is a gap in coverage. It isn't a pass.`, "");
+  }
+  if (target.npm) lines.push("**Archetypes.** A gap means the archetype wasn't tested, so it counts against coverage and never as a pass.", "", ...archetypeTable(target), "");
+  /** @type {Set<string>} */
+  const skipped = new Set();
+  for (const [name, archetype] of Object.entries(target.archetypes)) {
+    for (const config of archetype.configs) {
+      if (name !== "page") lines.push(`#### ${name}${configLabel(config)}.`, "");
+      for (const [tier, result] of Object.entries(config.tiers)) {
+        if (tier === "rules") {
+          for (const [engine, engineResult] of Object.entries(result.engines ?? {})) lines.push(engineSection(engine, engineResult, name !== "page"), "");
+        } else if (tier === "vsr" && result.status === "ran") {
+          lines.push(vsrSection(result, name !== "page"), "");
+        } else if (tier === "interactions" && result.status === "ran") {
+          lines.push(interactionsSection(result, name !== "page"), "");
+        } else if (result.status !== "ran") {
+          skipped.add(`${TIER_NAMES[tier] ?? tier}: ${sentence(result.reason ?? result.status)}.`);
+        }
+      }
+    }
+  }
+  if (skipped.size) lines.push("**Not run.**", "", ...[...skipped].map((item) => `- ${item}`), "");
+  return lines.join("\n");
+}
+
+export function failCheckSection(fail) {
+  if (!fail) return "";
+  const lines = ["## Fail check.", "", `Mode: ${fail.mode}. Each engine counts against its own threshold, and the counts aren't added together.`, ""];
+  if (fail.axe) lines.push(`- axe-core, impact ${fail.axe.threshold} or higher: ${plural(fail.axe.hits, "violation")}${fail.axe.tripped ? " (tripped)" : ""}.`);
+  if (fail.ibm) lines.push(`- IBM Equal Access, Toolkit level ${fail.ibm.threshold} or lower: ${plural(fail.ibm.hits, "violation")}${fail.ibm.tripped ? " (tripped)" : ""}.`);
+  lines.push("", `Result: ${fail.tripped ? "the fail check tripped." : "the fail check didn't trip."}`, "");
+  return lines.join("\n");
+}
+
+
+/** Top of every report: date, settings, tool versions, targets, and the warnings that apply to the whole run. */
+export function reportHeader(plan, results, title) {
+  const o = plan.options;
+  const lines = [
+    `# ${title}`,
+    "",
+    `Run date: ${results.runAt}.`,
+    "",
+    `Command: ${plan.command}. WCAG ${o.wcag}, level ${o.level}. Tiers: ${o.tiers.join(", ")}. Engines: ${o.engines.join(", ")}.`,
+    "",
+    "These results are a snapshot. The tools ran at their latest versions on this date, and a later run can differ.",
+    "",
+    "**Tool versions.**",
+    "",
+    ...toolLines(results.tools),
+    "",
+    "**Targets.**",
+    "",
+    ...plan.targets.map((t) => `- ${t.label}: ${t.resolved ? Object.values(t.resolved).filter(Boolean).join(" ") : t.input}${t.status === "failed" ? ` (failed: ${sentence(t.reason)})` : ""}`),
+    "",
+  ];
+  const evidence = new Set(plan.targets.filter((t) => t.evidenceLevel).map((t) => t.evidenceLevel));
+  if (evidence.size > 1) {
+    lines.push("**Warning.** This comparison mixes component evidence and page evidence. They test different things, so the results aren't equivalent.", "");
+  }
+  for (const warning of results.warnings) lines.push(`Warning: ${warning}`, "");
+  return lines;
+}
+
+/** What each target couldn't test, listed under the coverage section. */
+export function notTestableLines(results) {
+  const lines = [];
+  for (const target of results.targets) {
+    const items = target.summary.notTestable;
+    if (!items.length) continue;
+    lines.push(`**Not testable in ${target.id}.**`, "", ...items.slice(0, 20).map((item) => `- ${item}`), ...(items.length > 20 ? [`- and ${num(items.length - 20)} more`] : []), "");
+  }
+  return lines;
+}
+
+export const FINDINGS_NOTE = "Findings from different engines are listed separately. The engines overlap, and each catches things the other misses, so don't add their counts together. Impact is axe-core's own label. IBM Toolkit level is IBM's staged adoption scale (level 1 is essential requirements with high user impact). The two scales aren't comparable.";
+
+/** The fail check and the method note that close every report. */
+export function reportFooter(results) {
+  const lines = [];
+  const fail = failCheckSection(results.failCheck);
+  if (fail) lines.push(fail);
+  lines.push(
+    "## Method note.",
+    "",
+    "Automated rules cover only part of WCAG. A result of no automated violations found doesn't show that a page conforms to WCAG or works for everyone.",
+    "",
+    "A person has to check what these tools can't judge: whether alt text is meaningful, whether link and heading text make sense in context, cognitive load, reading order and focus order in real use, and how real screen readers behave.",
+    "",
+    "Contrast results depend on how the browser rendered the page, so the browser version is recorded above.",
+    "",
+  );
+  return lines;
+}
+
+/** Join lines into the final document. */
+export const finish = (lines) => `${lines.join("\n").replace(/\n{3,}/g, "\n\n")}\n`;

package/src/report/single.js

@@ -0,0 +1,16 @@
+import { FINDINGS_NOTE, coverageMatrix, finish, failCheckSection, notTestableLines, reportFooter, reportHeader, targetSection } from "./parts.js";
+
+/**
+ * Render report.md for an audit. The text comes from results.json and nothing else.
+ * @param {{ plan: any, results: any }} input
+ */
+export function renderSingleReport({ plan, results }) {
+  const lines = reportHeader(plan, results, "Accessibility report.");
+  lines.push("## Coverage.", "", coverageMatrix(plan, results), "");
+  lines.push("A gap, a failed engine, or a target that wasn't testable is a finding. It never counts as a pass.", "");
+  lines.push(...notTestableLines(results));
+  lines.push("## Findings.", "", FINDINGS_NOTE, "");
+  for (const target of results.targets) lines.push(targetSection(plan.targets.find((t) => t.id === target.id), target));
+  lines.push(...reportFooter(results));
+  return finish(lines);
+}

package/src/run/audit-npm.js

@@ -0,0 +1,279 @@
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { bundleEntries } from "../harness/bundle.js";
+import { installPackage } from "../harness/npm-install.js";
+import * as react from "../harness/npm-react.js";
+import * as wc from "../harness/npm-wc.js";
+import { closedShadowHosts, notTestableEntries } from "../harness/shadow.js";
+import { serveStatic } from "../harness/static-serve.js";
+import { openPage } from "../harness/url.js";
+import { candidateMapping, findAuthoredFixture } from "../plan/mapping.js";
+import { ARCHETYPES } from "../schema.js";
+import { runInteractions } from "../tiers/interactions/index.js";
+import { runRules } from "../tiers/rules/index.js";
+import { failedVsr, runVsr } from "../tiers/vsr.js";
+import { num } from "../text.js";
+import { summarize } from "./summary.js";
+
+/** The states worth checking for each archetype. The first is where a page starts. */
+const STATES = {
+  dialog: ["closed", "open"],
+  menu: ["closed", "open"],
+  tooltip: ["closed", "open"],
+  combobox: ["closed", "open"],
+  accordion: ["collapsed", "expanded"],
+};
+
+const firstLine = (error) => (error instanceof Error ? error.message : String(error)).split("\n").find((l) => l.trim()) ?? "unknown error";
+
+/** Page and console errors that mean a fixture didn't mount cleanly. A missing favicon doesn't count. */
+function watchErrors(errors) {
+  return (page) => {
+    page.on("pageerror", (error) => errors.push(error.message.split("\n")[0]));
+    page.on("console", (message) => {
+      if (message.type() !== "error") return;
+      const text = message.text();
+      if (/favicon|Failed to load resource/i.test(text) && !/\.(js|css)\b/.test(text)) return;
+      errors.push(text.split("\n")[0]);
+    });
+  };
+}
+
+/** Load the whole package in a page to list its exports and the custom elements it defines. */
+async function discover({ browser, workDir, flavor, pkg, buildDir }) {
+  const helper = flavor === "react" ? react : wc;
+  const entryFile = join(workDir, "discover.js");
+  writeFileSync(entryFile, helper.discoverEntry(pkg));
+  await bundleEntries({ entries: { discover: entryFile }, outdir: buildDir, workDir, react: flavor === "react" });
+  const server = await serveStatic(buildDir);
+  const errors = [];
+  try {
+    const opened = await openPage(browser, `${server.origin}/discover.html`, { beforeGoto: watchErrors(errors) });
+    try {
+      await opened.page.waitForFunction(() => /** @type {any} */ (window).__a11yExports !== undefined, undefined, { timeout: 15_000 });
+    } catch {
+      throw new Error(`The package didn't finish loading in the browser${errors.length ? `: ${errors[0]}` : "."}`);
+    }
+    const exportsList = await opened.page.evaluate(() => /** @type {any} */ (window).__a11yExports);
+    const tags = await opened.page.evaluate(() => [.../** @type {any} */ (window).__a11yDefined ?? []]);
+    await opened.close();
+    return { exports: exportsList, tags };
+  } finally {
+    await server.close();
+  }
+}
+
+/** Open one fixture page, check it follows the contract, and run the tiers in each state. `libA11y` is `on`, `off`, or `n/a`. */
+async function auditFixturePage({ browser, url, archetype, plan, libA11y }) {
+  const errors = [];
+  const opened = await openPage(browser, libA11y === "n/a" ? url : `${url}?libA11y=${libA11y}`, { beforeGoto: watchErrors(errors) });
+  try {
+    const trigger = opened.page.locator("[data-a11y-trigger]");
+    try {
+      await trigger.first().waitFor({ state: "attached", timeout: 5000 });
+    } catch {
+      return { gap: `The fixture didn't render an element with data-a11y-trigger${errors.length ? `: ${errors[0]}` : "."}` };
+    }
+    const count = await trigger.count();
+    if (count !== 1) return { gap: `The fixture must mark exactly one data-a11y-trigger. It marked ${num(count)}.` };
+    if (errors.length) return { gap: `The fixture logged errors when it mounted: ${errors[0]}` };
+
+    const states = STATES[archetype] ?? ["initial"];
+    const configs = [];
+    for (const [index, state] of states.entries()) {
+      let failure = null;
+      if (index > 0) {
+        try {
+          if (archetype === "tooltip") await trigger.first().focus();
+          else await trigger.first().click();
+          await opened.page.waitForFunction(
+            () => {
+              const root = document.querySelector("[data-a11y-root]");
+              const shown = root && /** @type {HTMLElement} */ (root).getClientRects().length > 0;
+              return shown || document.querySelector('[data-a11y-trigger][aria-expanded="true"]') !== null;
+            },
+            undefined,
+            { timeout: 3000 },
+          );
+        } catch {
+          failure = `The ${state} state never appeared after activating the trigger. A fixture's data-a11y-root has to show up when the ${archetype} opens.`;
+        }
+      }
+      await opened.page.evaluate(() => new Promise((done) => requestAnimationFrame(() => requestAnimationFrame(done))));
+      /** @type {Record<string, any>} */
+      const tiers = {};
+      for (const tier of plan.options.tiers) {
+        if (tier === "interactions") continue;
+        if (tier === "vsr") tiers.vsr = failure ? { status: "skipped", simulated: true, reason: failure } : await runVsr(opened.page, { scope: "body", state }).catch(failedVsr);
+        else if (failure) tiers.rules = { status: "failed", reason: failure, engines: Object.fromEntries(plan.options.engines.map((e) => [e, { status: "failed", reason: failure }])) };
+        else {
+          tiers.rules = await runRules(opened.page, { engines: plan.options.engines, wcag: plan.options.wcag, level: plan.options.level, scope: index === 0 ? "#root" : ["#root", "[data-a11y-root]"] });
+        }
+      }
+      configs.push({ libA11y, state, tiers });
+      if (errors.length) return { gap: `The fixture logged errors in the ${state} state: ${errors[0]}` };
+    }
+    const hidden = notTestableEntries(await closedShadowHosts(opened.page));
+    // The checks open their own fresh pages, so run them after this page's rules results are in.
+    if (plan.options.tiers.includes("interactions")) configs[0].tiers.interactions = await runInteractions(browser, libA11y === "n/a" ? url : `${url}?libA11y=${libA11y}`, archetype);
+    return { configs, hidden };
+  } finally {
+    await opened.close();
+  }
+}
+
+/**
+ * Audit one fixture. A library that ships opt-in accessibility features runs once for each `--lib-a11y` value, and each result carries its label.
+ * @param {{ browser: any, url: string, archetype: string, plan: any, toggle: boolean }} options
+ */
+async function auditFixture({ browser, url, archetype, plan, toggle }) {
+  const values = toggle ? plan.options.libA11y : ["n/a"];
+  const configs = [];
+  const hidden = [];
+  for (const libA11y of values) {
+    const outcome = await auditFixturePage({ browser, url, archetype, plan, libA11y });
+    if (outcome.gap) return { gap: toggle ? `With library accessibility ${libA11y}: ${outcome.gap}` : outcome.gap };
+    configs.push(...outcome.configs);
+    hidden.push(...outcome.hidden);
+  }
+  return { configs, hidden };
+}
+
+/**
+ * Audit an npm package: install it on its own, find what it exports, and audit each archetype that has a fixture.
+ * An archetype without a usable fixture is a gap with a reason, never a pass.
+ * @returns {Promise<{ result: any, mapping: Record<string, any> | null }>}
+ */
+export async function auditNpm({ browser, planTarget, plan, cwd, install = installPackage }) {
+  install ??= installPackage;
+  const base = { id: planTarget.id, reason: null, archetypes: {}, summary: { engines: {}, gaps: [], notTestable: [] }, warnings: [] };
+  const resolved = planTarget.resolved ?? {};
+  if (planTarget.kind === "npm-unsupported") {
+    return { result: { ...base, status: "unsupported", reason: `${resolved.framework ?? "This framework"} packages aren't supported. v1 covers React and web components.` }, mapping: null };
+  }
+  const tmp = mkdtempSync(join(tmpdir(), "automatica11y-npm-"));
+  const workDir = join(tmp, "install");
+  const buildDir = join(tmp, "build");
+  const fixtureDir = join(tmp, "fixtures");
+  mkdirSync(fixtureDir, { recursive: true });
+  const servers = [];
+  try {
+    /** @type {"react" | "wc" | "unknown"} */
+    let flavor = planTarget.kind === "npm-react" ? "react" : planTarget.kind === "npm-wc" ? "wc" : "unknown";
+    const installed = await install({ dir: workDir, name: resolved.name, version: resolved.version, flavor });
+    const warnings = [...installed.warnings];
+
+    const found = await discover({ browser, workDir, flavor: flavor === "react" ? "react" : "wc", pkg: resolved.name, buildDir });
+    if (flavor === "unknown") {
+      if (found.tags.length > 0) flavor = "wc";
+      else if (installed.react && found.exports.some((e) => /^[A-Z]/.test(e.name))) flavor = "react";
+    }
+    if (flavor === "unknown" || (flavor === "wc" && found.tags.length === 0)) {
+      return { result: { ...base, status: "not-applicable", reason: "The package has no rendering surface. It exports no React components and defines no custom elements.", warnings }, mapping: null };
+    }
+
+    const candidates = candidateMapping({ flavor, exports: found.exports, tags: found.tags });
+    const kindFlavor = /** @type {"react" | "wc"} */ (flavor);
+    const wanted = plan.options.archetypes ?? ARCHETYPES;
+    /** @type {Record<string, any>} */
+    const mapping = {};
+    /** @type {Record<string, any>} */
+    const archetypes = {};
+    const gaps = [];
+    const hidden = [];
+    const helper = flavor === "react" ? react : wc;
+
+    // Decide where each archetype's fixture comes from, then bundle each one on its own so one bad fixture can't break the rest.
+    const runnable = {};
+    for (const archetype of wanted) {
+      const user = planTarget.mapping?.[archetype] ?? {};
+      const entry = { ...candidates[archetype], ...user };
+      const authored = findAuthoredFixture({ cwd, targetId: planTarget.id, archetype, mapped: user });
+      let fixture = null;
+      if (authored) {
+        entry.status = "authored";
+        entry.fixture = authored;
+        delete entry.reason;
+        fixture = authored;
+      } else if (user.fixture) {
+        entry.status = "needs-fixture";
+        entry.reason = `The mapping names ${user.fixture}, but that file doesn't exist.`;
+      } else if (entry.status === "template" || (user.export || user.tag) && ["button", "link"].includes(archetype)) {
+        const source = flavor === "react" ? react.template(archetype, resolved.name, entry.export) : wc.template(archetype, entry.tag);
+        if (source) {
+          entry.status = "template";
+          delete entry.reason;
+          fixture = join(fixtureDir, `${archetype}.${flavor === "react" ? "jsx" : "js"}`);
+          writeFileSync(fixture, source);
+        }
+      }
+      mapping[archetype] = entry;
+      if (!fixture) {
+        const reason = entry.status === "no-match" ? entry.reason : entry.reason ?? `The ${archetype} archetype needs a fixture someone writes.`;
+        archetypes[archetype] = { status: "gap", reason: archetype && entry.status !== "no-match" ? `${reason} Write fixtures/${planTarget.id}/${archetype}.${flavor === "react" ? "jsx" : "js"}.` : reason, configs: [] };
+        gaps.push(`archetype:${archetype}`);
+        continue;
+      }
+      const entryFile = join(tmp, "entries", `${archetype}-entry.js`);
+      mkdirSync(join(tmp, "entries"), { recursive: true });
+      writeFileSync(entryFile, helper.entry(fixture, resolved.name));
+      try {
+        await bundleEntries({ entries: { [archetype]: entryFile }, outdir: buildDir, workDir, react: flavor === "react" });
+        runnable[archetype] = `/${archetype}.html`;
+      } catch (error) {
+        archetypes[archetype] = { status: "gap", reason: `The fixture didn't bundle. ${firstLine(error)}`, configs: [] };
+        gaps.push(`archetype:${archetype}`);
+        entry.status = "needs-fixture";
+        entry.reason = firstLine(error);
+      }
+    }
+
+    if (Object.keys(runnable).length > 0) {
+      const live = await serveStatic(buildDir);
+      servers.push(live);
+      for (const [archetype, path] of Object.entries(runnable)) {
+        /** @type {any} */
+        const outcome = await auditFixture({ browser, url: `${live.origin}${path}`, archetype, plan, toggle: mapping[archetype].libA11y === true }).catch((error) => ({ gap: firstLine(error) }));
+        if (outcome.gap) {
+          archetypes[archetype] = { status: "gap", reason: outcome.gap, configs: [] };
+          gaps.push(`archetype:${archetype}`);
+          mapping[archetype].status = "needs-fixture";
+          mapping[archetype].reason = outcome.gap;
+        } else {
+          archetypes[archetype] = { status: "ran", configs: outcome.configs };
+          hidden.push(...outcome.hidden.map((h) => `${archetype}: ${h}`));
+        }
+      }
+    }
+
+    const ordered = Object.fromEntries(wanted.filter((a) => archetypes[a]).map((a) => [a, archetypes[a]]));
+    const ranAny = Object.values(ordered).some((a) => a.status === "ran");
+    return {
+      result: {
+        id: planTarget.id,
+        status: ranAny ? "ran" : "failed",
+        reason: ranAny ? null : "No archetype had a usable fixture, so nothing was tested. See the gaps for what each one needs.",
+        archetypes: ordered,
+        npm: {
+          name: resolved.name,
+          version: installed.version ?? resolved.version,
+          flavor,
+          framework: resolved.framework ?? null,
+          react: installed.react,
+          reactDom: installed.reactDom,
+          tags: flavor === "wc" ? found.tags : [],
+        },
+        summary: (() => {
+          const base = summarize(ordered, plan.options.engines, gaps);
+          return { ...base, notTestable: [...base.notTestable, ...hidden] };
+        })(),
+        warnings,
+      },
+      mapping: Object.fromEntries(Object.entries(mapping).map(([k, v]) => [k, { ...v, fixture: v.fixture ?? null }])),
+    };
+  } finally {
+    for (const s of servers) await s.close();
+    if (!process.env.AUTOMATICA11Y_KEEP_TEMP) rmSync(tmp, { recursive: true, force: true });
+  }
+}