automatica11y 0.0.0-stage → 0.3.0

package/src/run/fail-check.js

@@ -0,0 +1,62 @@
+import { IMPACTS } from "../schema.js";
+
+const RANK = Object.fromEntries(IMPACTS.map((impact, index) => [impact, index]));
+
+/** All findings an engine reported for one target, across every archetype and configuration. */
+function violationsFor(target, engine) {
+  const found = [];
+  for (const archetype of Object.values(target.archetypes)) {
+    for (const config of archetype.configs) {
+      const result = config.tiers.rules?.engines?.[engine];
+      if (result?.status === "ran") found.push(...(result.violations ?? []));
+    }
+  }
+  return found;
+}
+
+/** Count axe violations at or above the impact threshold. */
+export function countAxeHits(target, threshold) {
+  return violationsFor(target, "axe").filter((finding) => finding.impact && RANK[finding.impact] >= RANK[threshold]).length;
+}
+
+/** Count IBM violations whose Toolkit level is at or below the threshold. */
+export function countIbmHits(target, threshold) {
+  return violationsFor(target, "ibm").filter((finding) => finding.toolkitLevel != null && finding.toolkitLevel <= threshold).length;
+}
+
+/**
+ * Decide whether the run trips its fail check. Each engine has its own threshold, and counts are never added across engines.
+ * `any` trips a target when any checked engine hits. `all` trips it only when every checked engine hits.
+ * A target that failed, or an engine that failed, counts as no hit.
+ * @param {Array<any>} targets Target results.
+ * @param {{ mode: string, axe: string | null, ibm: number | null } | null} fail
+ */
+export function evaluateFailCheck(targets, fail) {
+  if (!fail) return null;
+  const checked = [fail.axe ? "axe" : null, fail.ibm ? "ibm" : null].filter(Boolean);
+  const totals = { axe: fail.axe ? { threshold: fail.axe, hits: 0, tripped: false } : null, ibm: fail.ibm ? { threshold: fail.ibm, hits: 0, tripped: false } : null };
+  const perTarget = {};
+  let tripped = false;
+  for (const target of targets) {
+    const entry = { axe: null, ibm: null, tripped: false };
+    if (target.status === "ran") {
+      if (fail.axe && totals.axe) {
+        const hits = countAxeHits(target, fail.axe);
+        entry.axe = { threshold: fail.axe, hits, tripped: hits > 0 };
+        totals.axe.hits += hits;
+        totals.axe.tripped ||= hits > 0;
+      }
+      if (fail.ibm && totals.ibm) {
+        const hits = countIbmHits(target, fail.ibm);
+        entry.ibm = { threshold: fail.ibm, hits, tripped: hits > 0 };
+        totals.ibm.hits += hits;
+        totals.ibm.tripped ||= hits > 0;
+      }
+      const flags = checked.map((engine) => entry[/** @type {"axe" | "ibm"} */ (engine)]?.tripped === true);
+      entry.tripped = fail.mode === "all" ? flags.every(Boolean) : flags.some(Boolean);
+    }
+    perTarget[target.id] = entry;
+    tripped ||= entry.tripped;
+  }
+  return { mode: fail.mode, axe: totals.axe, ibm: totals.ibm, targets: perTarget, tripped };
+}

package/src/run/pool.js

@@ -0,0 +1,21 @@
+/**
+ * Run `fn` over `items` with at most `limit` in flight. Results keep the order of `items`.
+ * @template T, R
+ * @param {T[]} items
+ * @param {number} limit
+ * @param {(item: T, index: number) => Promise<R>} fn
+ * @returns {Promise<R[]>}
+ */
+export async function mapPool(items, limit, fn) {
+  const results = new Array(items.length);
+  let next = 0;
+  const worker = async () => {
+    while (next < items.length) {
+      const index = next;
+      next += 1;
+      results[index] = await fn(items[index], index);
+    }
+  };
+  await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker));
+  return results;
+}

package/src/run/run-plan.js

@@ -0,0 +1,262 @@
+import { existsSync, mkdirSync, writeFileSync } from "node:fs";
+import { basename, dirname, resolve } from "node:path";
+import { checkEnvironment } from "../env/browser.js";
+import { readToolVersions } from "../env/versions.js";
+import { launchBrowser } from "../harness/browser.js";
+import { serveStatic } from "../harness/static-serve.js";
+import { listStories, readIndex, selectStories, storyUrl, waitForStory } from "../harness/storybook.js";
+import { closedShadowHosts, notTestableEntries } from "../harness/shadow.js";
+import { NOT_APPLICABLE_FOR_PAGES } from "../tiers/interactions/index.js";
+import { failedVsr, runVsr } from "../tiers/vsr.js";
+import { openPage } from "../harness/url.js";
+import { renderReport } from "../report/index.js";
+import { num } from "../text.js";
+import { parseResults } from "../schema.js";
+import { runRules, selfTest } from "../tiers/rules/index.js";
+import { evaluateFailCheck } from "./fail-check.js";
+import { mapPool } from "./pool.js";
+import { auditNpm } from "./audit-npm.js";
+import { failedTarget, summarize } from "./summary.js";
+
+/** How many stories to audit at once. */
+const STORY_CONCURRENCY = 4;
+
+const EXIT = { OK: 0, FAIL_THRESHOLD: 1, ENVIRONMENT: 3, ALL_TARGETS_FAILED: 4 };
+
+const NPM_KINDS = new Set(["npm", "npm-react", "npm-wc", "npm-unsupported"]);
+const UNSUPPORTED_KIND = (kind) => `${kind} targets aren't supported.`;
+
+/** Audit one page and return its target result. */
+async function auditPage(browser, url, planTarget, plan, extraWarnings) {
+  const opened = await openPage(browser, url);
+  try {
+    /** @type {Record<string, any>} */
+    const tiers = {};
+    for (const tier of plan.options.tiers) {
+      if (tier === "rules") {
+        tiers.rules = await runRules(opened.page, { engines: plan.options.engines, wcag: plan.options.wcag, level: plan.options.level });
+      } else if (tier === "interactions") {
+        tiers.interactions = NOT_APPLICABLE_FOR_PAGES;
+      } else {
+        tiers.vsr = await runVsr(opened.page, { scope: "body" }).catch(failedVsr);
+      }
+    }
+    const archetypes = { page: { status: "ran", configs: [{ libA11y: "n/a", tiers }] } };
+    const hidden = notTestableEntries(await closedShadowHosts(opened.page));
+    const summary = summarize(archetypes, plan.options.engines);
+    summary.notTestable = [...summary.notTestable, ...hidden];
+    return {
+      id: planTarget.id,
+      status: "ran",
+      reason: null,
+      archetypes,
+      summary,
+      warnings: [...extraWarnings, ...opened.warnings, ...hidden.map((h) => `Not testable: ${h}.`)],
+    };
+  } finally {
+    await opened.close();
+  }
+}
+
+/** Audit one story in its own page. The story's root element is the scope, so page-level rules don't fire. */
+async function auditStory(browser, base, story, plan) {
+  const opened = await openPage(browser, storyUrl(base, story.id));
+  try {
+    await waitForStory(opened.page);
+    /** @type {Record<string, any>} */
+    const tiers = {};
+    for (const tier of plan.options.tiers) {
+      tiers[tier] =
+        tier === "rules"
+          ? await runRules(opened.page, { engines: plan.options.engines, wcag: plan.options.wcag, level: plan.options.level, scope: "#storybook-root" })
+          : tier === "interactions"
+            ? NOT_APPLICABLE_FOR_PAGES
+            : await runVsr(opened.page, { scope: "#storybook-root" }).catch(failedVsr);
+    }
+    const hidden = notTestableEntries(await closedShadowHosts(opened.page));
+    return { id: story.id, ok: true, archetype: { status: "ran", configs: [{ libA11y: "n/a", tiers }] }, hidden };
+  } catch (error) {
+    return { id: story.id, ok: false, reason: error instanceof Error ? error.message.split("\n")[0] : String(error) };
+  } finally {
+    await opened.close();
+  }
+}
+
+/** Audit a Storybook: list the stories, pick the ones to run, and audit each as its own unit. */
+async function auditStorybook(browser, planTarget, plan, servers) {
+  const resolved = planTarget.resolved;
+  const index = await readIndex(resolved);
+  const stories = listStories(index);
+  if (stories.length === 0) return failedTarget(planTarget.id, "The Storybook index lists no stories.");
+  const picked = selectStories(stories, { archetypes: plan.options.archetypes, max: plan.options.maxStories });
+  if (picked.selected.length === 0) {
+    return failedTarget(planTarget.id, `No stories matched the archetypes ${plan.options.archetypes.join(", ")}.`);
+  }
+  let base = resolved.url;
+  if (resolved.path) {
+    const server = await serveStatic(resolved.path);
+    servers.push(server);
+    base = `${server.origin}/`;
+  }
+  const audited = await mapPool(picked.selected, STORY_CONCURRENCY, (story) => auditStory(browser, base, story, plan));
+
+  const archetypes = {};
+  const failedStories = [];
+  for (const item of audited) {
+    if (item.ok) archetypes[`story:${item.id}`] = item.archetype;
+    else {
+      archetypes[`story:${item.id}`] = { status: "gap", configs: [] };
+      failedStories.push({ id: item.id, reason: item.reason });
+    }
+  }
+  const warnings = [];
+  if (picked.truncated) {
+    warnings.push(`The story cap cut the list short. ${num(picked.selected.length)} of ${num(picked.matched)} ${plan.options.archetypes ? "matching " : ""}stories ${picked.selected.length === 1 ? "was" : "were"} audited, spread across components. Raise --max-stories to audit more.`);
+  }
+  const gaps = failedStories.map((f) => `story:${f.id}`);
+  for (const archetype of plan.options.archetypes ?? []) {
+    if (!picked.matchedByArchetype[archetype]) {
+      gaps.push(`archetype:${archetype}`);
+      warnings.push(`No stories matched the ${archetype} archetype. That's a gap in coverage, not a pass.`);
+    }
+  }
+  const ranCount = audited.filter((item) => item.ok).length;
+  if (ranCount === 0) return failedTarget(planTarget.id, `None of the ${audited.length} audited stories rendered.`);
+  return {
+    id: planTarget.id,
+    status: "ran",
+    reason: null,
+    archetypes,
+    storybook: {
+      index: resolved.index,
+      total: picked.total,
+      matched: picked.matched,
+      audited: audited.length,
+      truncated: picked.truncated,
+      maxStories: plan.options.maxStories,
+      filtered: picked.filtered,
+      archetypeMatches: picked.matchedByArchetype,
+      failedStories,
+    },
+    summary: (() => {
+      const base = summarize(archetypes, plan.options.engines, gaps);
+      return { ...base, notTestable: [...base.notTestable, ...audited.flatMap((item) => (item.ok ? item.hidden.map((h) => `${item.id}: ${h}`) : []))] };
+    })(),
+    warnings,
+  };
+}
+
+/** Run one target. Anything that goes wrong becomes a failed result, so one target can't stop the rest. */
+async function runTarget(browser, planTarget, plan, io) {
+  if (planTarget.status === "failed") return { result: failedTarget(planTarget.id, planTarget.reason), mapping: null };
+  const servers = [];
+  try {
+    const pageNote = plan.options.archetypes ? ["--archetypes only applies to Storybook and npm targets. This page was checked as a whole."] : [];
+    if (planTarget.kind === "storybook") return { result: await auditStorybook(browser, planTarget, plan, servers), mapping: null };
+    if (NPM_KINDS.has(planTarget.kind)) return await auditNpm({ browser, planTarget, plan, cwd: io.cwd, install: io.installPackage });
+    if (planTarget.kind === "url") return { result: await auditPage(browser, planTarget.resolved.url, planTarget, plan, pageNote), mapping: null };
+    if (planTarget.kind === "html-file") {
+      const file = planTarget.resolved.path;
+      const server = await serveStatic(dirname(file));
+      servers.push(server);
+      return { result: await auditPage(browser, `${server.origin}/${encodeURIComponent(basename(file))}`, planTarget, plan, pageNote), mapping: null };
+    }
+    if (planTarget.kind === "static-dir") {
+      const dir = planTarget.resolved.path;
+      if (!existsSync(resolve(dir, "index.html"))) return { result: failedTarget(planTarget.id, "The directory has no index.html to check."), mapping: null };
+      const server = await serveStatic(dir);
+      servers.push(server);
+      return { result: await auditPage(browser, `${server.origin}/`, planTarget, plan, ["Only index.html was checked. Other pages in the directory weren't.", ...pageNote]), mapping: null };
+    }
+    return { result: failedTarget(planTarget.id, UNSUPPORTED_KIND(planTarget.kind)), mapping: null };
+  } catch (error) {
+    return { result: failedTarget(planTarget.id, error instanceof Error ? error.message.split("\n")[0] : String(error)), mapping: null };
+  } finally {
+    for (const server of servers) await server.close();
+  }
+}
+
+/** A short line per target for the terminal. */
+function describeResult(target) {
+  if (target.status !== "ran") return `${target.id}: ${target.status} (${target.reason})`;
+  const parts = Object.entries(target.summary.engines).map(([engine, s]) => {
+    if (s.status !== "ran") return `${engine} ${s.status}`;
+    const extra = engine === "axe" ? Object.entries(s.violationsByImpact).filter(([, n]) => n).map(([k, n]) => `${n} ${k}`) : [];
+    return `${engine} ${s.violations} violation${s.violations === 1 ? "" : "s"}${extra.length ? ` (${extra.join(", ")})` : ""}, ${s.needsReview} to review`;
+  });
+  return `${target.id}: ${parts.join("; ") || "no engines ran"}`;
+}
+
+/**
+ * Run a plan: check the environment, launch one browser, audit each target, write results.json and report.md.
+ * @param {any} plan
+ * @param {import("../commands/common.js").Io} io
+ * @returns {Promise<number>} The exit code.
+ */
+export async function runPlan(plan, io) {
+  const environment = await checkEnvironment({ env: io.env, platform: io.platform });
+  if (!environment.ok) {
+    for (const problem of environment.problems) io.stderr.write(`${problem.message}\n  Fix: ${problem.fix}\n`);
+    return EXIT.ENVIRONMENT;
+  }
+  let session;
+  try {
+    session = await launchBrowser({ env: io.env, platform: io.platform });
+  } catch (error) {
+    io.stderr.write(`Couldn't start the browser: ${error instanceof Error ? error.message.split("\n")[0] : String(error)}\n  Run "automatica11y doctor" to check the setup.\n`);
+    return EXIT.ENVIRONMENT;
+  }
+  const { browser, version } = session;
+  try {
+    if (plan.options.tiers.includes("rules")) {
+      const test = await selfTest(browser, plan.options.engines);
+      if (!test.passed) {
+        io.stderr.write(`The rule engines failed their self-test, so no results are reported.\n${test.problems.map((p) => `  ${p}`).join("\n")}\n`);
+        return EXIT.ENVIRONMENT;
+      }
+    }
+    const targets = [];
+    const mappings = {};
+    for (const planTarget of plan.targets) {
+      const { result, mapping } = await runTarget(browser, planTarget, plan, io);
+      targets.push(result);
+      if (mapping) {
+        mappings[planTarget.id] = mapping;
+        planTarget.mapping = mapping;
+      }
+      if (result.npm) planTarget.kind = result.npm.flavor === "react" ? "npm-react" : "npm-wc";
+    }
+
+    const results = parseResults({
+      schema: 1,
+      planRef: "plan.json",
+      runAt: new Date().toISOString(),
+      tools: readToolVersions({ chromium: version }),
+      targets,
+      failCheck: evaluateFailCheck(targets, plan.options.fail),
+      warnings: [],
+    });
+    const outDir = resolve(io.cwd, plan.options.out);
+    mkdirSync(outDir, { recursive: true });
+    writeFileSync(resolve(outDir, "results.json"), `${JSON.stringify(results, null, 2)}\n`);
+    writeFileSync(resolve(outDir, "report.md"), renderReport({ plan, results }));
+    if (Object.keys(mappings).length > 0) {
+      // The candidate mapping, in the shape --mapping reads, so it can be edited and passed back in.
+      writeFileSync(resolve(outDir, "mapping.json"), `${JSON.stringify(mappings, null, 2)}\n`);
+      writeFileSync(resolve(outDir, "plan.json"), `${JSON.stringify(plan, null, 2)}\n`);
+    }
+
+    const lines = ["", ...results.targets.map((t) => `  ${describeResult(t)}`)];
+    if (results.failCheck) lines.push(`  Fail check (${results.failCheck.mode}): ${results.failCheck.tripped ? "tripped" : "not tripped"}`);
+    lines.push(`  Wrote ${resolve(outDir, "results.json")}`, `  Wrote ${resolve(outDir, "report.md")}`);
+    io.stdout.write(`${lines.join("\n")}\n`);
+
+    if (!results.targets.some((t) => t.status === "ran")) {
+      io.stderr.write("No target produced results.\n");
+      return EXIT.ALL_TARGETS_FAILED;
+    }
+    return results.failCheck?.tripped ? EXIT.FAIL_THRESHOLD : EXIT.OK;
+  } finally {
+    await browser.close().catch(() => {});
+  }
+}

package/src/run/summary.js

@@ -0,0 +1,49 @@
+/** Roll the engine results up into counts. Impact counts belong to axe and Toolkit-level counts belong to IBM. */
+export function summarize(archetypes, engines, gaps = []) {
+  /** @type {any} */
+  const summary = { engines: {}, gaps, notTestable: [] };
+  for (const engine of engines) {
+    const results = Object.values(archetypes).flatMap((a) => a.configs.map((c) => c.tiers.rules?.engines?.[engine]).filter(Boolean));
+    if (results.length === 0) continue;
+    const ran = results.filter((r) => r.status === "ran");
+    const entry = {
+      status: ran.length ? "ran" : results[0].status,
+      violations: ran.reduce((n, r) => n + r.violations.length, 0),
+      needsReview: ran.reduce((n, r) => n + r.incomplete.length, 0),
+    };
+    if (engine === "axe") {
+      entry.violationsByImpact = { critical: 0, serious: 0, moderate: 0, minor: 0 };
+      for (const r of ran) for (const f of r.violations) if (f.impact) entry.violationsByImpact[f.impact] += 1;
+    }
+    if (engine === "ibm") {
+      entry.violationsByToolkitLevel = { 1: 0, 2: 0, 3: 0, 4: 0 };
+      for (const r of ran) for (const f of r.violations) if (f.toolkitLevel != null) entry.violationsByToolkitLevel[f.toolkitLevel] += 1;
+    }
+    summary.engines[engine] = entry;
+  }
+  for (const [name, archetype] of Object.entries(archetypes)) {
+    for (const config of archetype.configs) {
+      const rules = config.tiers.rules;
+      if (rules?.status === "not-testable") summary.notTestable.push(`${name === "page" ? "" : `${name}: `}${rules.reason}`);
+    }
+  }
+  const checks = Object.values(archetypes).flatMap((a) => a.configs.flatMap((c) => c.tiers.interactions?.checks ?? []));
+  if (checks.length) {
+    summary.interactions = { pass: 0, fail: 0, notApplicable: 0, error: 0 };
+    for (const check of checks) summary.interactions[check.result === "not-applicable" ? "notApplicable" : check.result] += 1;
+  }
+  const walks = Object.entries(archetypes).flatMap(([name, a]) => a.configs.map((c) => ({ name, vsr: c.tiers.vsr })).filter((x) => x.vsr?.status === "ran"));
+  if (walks.length) {
+    summary.vsr = { walks: walks.length, flagged: walks.reduce((n, w) => n + w.vsr.flags.length, 0) };
+    const seen = new Set();
+    for (const { name, vsr } of walks) for (const entry of vsr.notTestable ?? []) {
+      const line = name === "page" ? entry : `${name}: ${entry}`;
+      if (!seen.has(line)) { seen.add(line); summary.notTestable.push(line); }
+    }
+  }
+  return summary;
+}
+
+export function failedTarget(id, reason) {
+  return { id, status: "failed", reason, archetypes: {}, summary: { engines: {}, gaps: [], notTestable: [] }, warnings: [] };
+}

package/src/schema.js

@@ -0,0 +1,255 @@
+import * as v from "valibot";
+
+export const WCAG_VERSIONS = ["2.0", "2.1", "2.2"];
+export const LEVELS = ["A", "AA", "AAA"];
+export const TIERS = ["rules", "interactions", "vsr"];
+export const ENGINES = ["axe", "ibm"];
+export const LIB_A11Y = ["on", "off"];
+export const IMPACTS = ["minor", "moderate", "serious", "critical"];
+export const TOOLKIT_LEVELS = [1, 2, 3];
+export const FAIL_MODES = ["any", "all"];
+export const ARCHETYPES = ["button", "link", "dialog", "menu", "tabs", "combobox", "form-field", "accordion", "tooltip", "chart"];
+export const FLAVORS = ["react", "wc"];
+export const MAPPING_STATUSES = ["template", "authored", "needs-fixture", "no-match"];
+
+/** What a candidate mapping says about one archetype of one npm target. */
+export const MappingEntrySchema = v.object({
+  flavor: v.optional(v.picklist(FLAVORS)),
+  /** Export name (React) the fixture or template uses. */
+  export: v.optional(v.string()),
+  /** Custom element tag (web components) the fixture or template uses. */
+  tag: v.optional(v.string()),
+  /** Path to an authored fixture, relative to where the command runs. */
+  fixture: v.optional(v.nullable(v.string())),
+  /** The library ships opt-in accessibility features. The fixture gets `libA11y` (true or false) and `--lib-a11y` runs it both ways. */
+  libA11y: v.optional(v.boolean()),
+  status: v.optional(v.picklist(MAPPING_STATUSES)),
+  candidates: v.optional(v.array(v.string())),
+  parts: v.optional(v.array(v.string())),
+  reason: v.optional(v.string()),
+});
+
+/** The file `--mapping` points to: target id, then archetype. */
+export const MappingFileSchema = v.record(v.string(), v.record(v.picklist(ARCHETYPES), MappingEntrySchema));
+
+/**
+ * Parse a mapping file. Throws an Error with a readable summary when it's invalid.
+ * @param {unknown} input
+ */
+export function parseMappingFile(input) {
+  const result = v.safeParse(MappingFileSchema, input);
+  if (!result.success) throw new Error(`Invalid mapping:\n${v.summarize(result.issues)}`);
+  return result.output;
+}
+
+export const TARGET_KINDS = ["npm", "npm-react", "npm-wc", "npm-unsupported", "npm-non-ui", "storybook", "url", "html-file", "static-dir"];
+
+const nullableString = v.nullable(v.string());
+
+export const TargetSchema = v.object({
+  id: v.string(),
+  input: v.string(),
+  label: v.string(),
+  status: v.picklist(["ok", "failed"]),
+  reason: nullableString,
+  kind: v.nullable(v.picklist(TARGET_KINDS)),
+  evidenceLevel: v.nullable(v.picklist(["component", "page"])),
+  resolved: v.nullable(v.record(v.string(), nullableString)),
+  mapping: v.nullable(v.record(v.string(), MappingEntrySchema)),
+});
+
+export const FailConfigSchema = v.object({
+  mode: v.picklist(FAIL_MODES),
+  axe: v.nullable(v.picklist(IMPACTS)),
+  ibm: v.nullable(v.picklist(TOOLKIT_LEVELS)),
+});
+
+export const PlanSchema = v.object({
+  schema: v.literal(1),
+  createdAt: v.string(),
+  command: v.picklist(["audit", "compare"]),
+  options: v.object({
+    wcag: v.picklist(WCAG_VERSIONS),
+    level: v.picklist(LEVELS),
+    tiers: v.array(v.picklist(TIERS)),
+    engines: v.array(v.picklist(ENGINES)),
+    libA11y: v.array(v.picklist(LIB_A11Y)),
+    archetypes: v.nullable(v.array(v.picklist(ARCHETYPES))),
+    mapping: nullableString,
+    maxStories: v.pipe(v.number(), v.integer(), v.minValue(1)),
+    out: v.string(),
+    fail: v.nullable(FailConfigSchema),
+  }),
+  tools: v.record(v.string(), nullableString),
+  targets: v.array(TargetSchema),
+});
+
+/**
+ * Parse a plan. Throws an Error with a readable summary when the plan is invalid.
+ * @param {unknown} input
+ */
+export function parsePlan(input) {
+  const result = v.safeParse(PlanSchema, input);
+  if (!result.success) throw new Error(`Invalid plan:\n${v.summarize(result.issues)}`);
+  return result.output;
+}
+
+// ---- Results ----
+
+const TIER_STATUS = ["ran", "skipped", "not-applicable", "not-testable", "failed"];
+const IMPACT_COUNTS = v.object({ critical: v.number(), serious: v.number(), moderate: v.number(), minor: v.number() });
+
+const NodeSchema = v.object({ selector: v.string(), html: v.string() });
+
+/** One finding from one engine. `impact` belongs to axe. `toolkitLevel` belongs to IBM. Neither converts to the other. */
+const FindingSchema = v.object({
+  ruleId: v.string(),
+  impact: v.nullable(v.picklist(IMPACTS)),
+  toolkitLevel: v.optional(v.nullable(v.number())),
+  kind: v.optional(v.picklist(["potential", "manual", "recommendation"])),
+  wcag: v.array(v.string()),
+  tags: v.optional(v.array(v.string())),
+  help: v.string(),
+  helpUrl: v.string(),
+  nodeCount: v.number(),
+  nodes: v.array(NodeSchema),
+});
+
+const EngineResultSchema = v.object({
+  status: v.picklist(TIER_STATUS),
+  reason: v.optional(v.nullable(v.string())),
+  version: v.optional(v.nullable(v.string())),
+  config: v.optional(v.record(v.string(), v.unknown())),
+  violations: v.optional(v.array(FindingSchema)),
+  incomplete: v.optional(v.array(FindingSchema)),
+  /** Rules that passed. Both engines count rules, not elements. */
+  passesCount: v.optional(v.number()),
+  notes: v.optional(v.array(v.string())),
+});
+
+const TierResultSchema = v.object({
+  status: v.picklist(TIER_STATUS),
+  reason: v.optional(v.nullable(v.string())),
+  engines: v.optional(v.record(v.string(), EngineResultSchema)),
+  checks: v.optional(
+    v.array(
+      v.object({
+        name: v.string(),
+        criteria: v.optional(v.array(v.string())),
+        result: v.picklist(["pass", "fail", "not-applicable", "error"]),
+        detail: v.string(),
+        /** How the focus indicator was detected: computed-style or screenshot. */
+        method: v.optional(v.string()),
+      }),
+    ),
+  ),
+  simulated: v.optional(v.boolean()),
+  version: v.optional(v.nullable(v.string())),
+  log: v.optional(
+    v.array(
+      v.object({
+        state: v.nullable(v.optional(v.string())),
+        announcements: v.array(v.string()),
+        reachedEnd: v.boolean(),
+        truncated: v.boolean(),
+      }),
+    ),
+  ),
+  /** Phrases a person should look at: a control announced as only its role, or a role announced as generic. */
+  flags: v.optional(v.array(v.object({ type: v.string(), phrase: v.string(), index: v.number(), state: v.nullable(v.optional(v.string())) }))),
+  notTestable: v.optional(v.array(v.string())),
+  notes: v.optional(v.array(v.string())),
+});
+
+const EngineSummarySchema = v.object({
+  status: v.picklist(TIER_STATUS),
+  violations: v.number(),
+  needsReview: v.number(),
+  /** axe only. IBM findings have no impact. */
+  violationsByImpact: v.optional(IMPACT_COUNTS),
+  /** IBM only. Keys are Toolkit levels 1 to 4. */
+  violationsByToolkitLevel: v.optional(v.record(v.string(), v.number())),
+});
+
+const StorybookInfoSchema = v.object({
+  index: v.string(),
+  /** Stories in the index, before any filtering. */
+  total: v.number(),
+  /** Stories left after the archetype filter. */
+  matched: v.number(),
+  audited: v.number(),
+  /** True when the story cap cut the list short. */
+  truncated: v.boolean(),
+  maxStories: v.number(),
+  /** True when --archetypes limited which stories were audited. */
+  filtered: v.optional(v.boolean()),
+  /** Every archetype each story's title, name, and tags suggest, for all stories in the index. */
+  archetypeMatches: v.record(v.string(), v.array(v.string())),
+  /** Stories that didn't render. They count as failures, never as passes. */
+  failedStories: v.array(v.object({ id: v.string(), reason: v.string() })),
+});
+
+export const TargetResultSchema = v.object({
+  id: v.string(),
+  status: v.picklist(["ran", "failed", "unsupported", "not-applicable"]),
+  reason: nullableString,
+  archetypes: v.record(
+    v.string(),
+    v.object({
+      status: v.picklist(["ran", "gap"]),
+      reason: v.optional(v.nullable(v.string())),
+      configs: v.array(v.object({ libA11y: v.picklist(["on", "off", "n/a"]), state: v.optional(v.string()), tiers: v.record(v.string(), TierResultSchema) })),
+    }),
+  ),
+  storybook: v.optional(StorybookInfoSchema),
+  npm: v.optional(
+    v.object({
+      name: v.string(),
+      version: nullableString,
+      flavor: v.picklist(FLAVORS),
+      framework: nullableString,
+      react: nullableString,
+      reactDom: nullableString,
+      tags: v.array(v.string()),
+    }),
+  ),
+  summary: v.object({
+    engines: v.record(v.string(), EngineSummarySchema),
+    gaps: v.array(v.string()),
+    notTestable: v.array(v.string()),
+    interactions: v.optional(v.object({ pass: v.number(), fail: v.number(), notApplicable: v.number(), error: v.number() })),
+    vsr: v.optional(v.object({ walks: v.number(), flagged: v.number() })),
+  }),
+  warnings: v.array(v.string()),
+});
+
+const FailEngineSchema = v.object({ threshold: v.union([v.string(), v.number()]), hits: v.number(), tripped: v.boolean() });
+
+export const FailCheckSchema = v.object({
+  mode: v.picklist(FAIL_MODES),
+  axe: v.nullable(FailEngineSchema),
+  ibm: v.nullable(FailEngineSchema),
+  /** Per target, because the combined mode decides one target at a time. */
+  targets: v.record(v.string(), v.object({ axe: v.nullable(FailEngineSchema), ibm: v.nullable(FailEngineSchema), tripped: v.boolean() })),
+  tripped: v.boolean(),
+});
+
+export const ResultsSchema = v.object({
+  schema: v.literal(1),
+  planRef: v.string(),
+  runAt: v.string(),
+  tools: v.record(v.string(), nullableString),
+  targets: v.array(TargetResultSchema),
+  failCheck: v.nullable(FailCheckSchema),
+  warnings: v.array(v.string()),
+});
+
+/**
+ * Parse a results object. Throws an Error with a readable summary when it's invalid.
+ * @param {unknown} input
+ */
+export function parseResults(input) {
+  const result = v.safeParse(ResultsSchema, input);
+  if (!result.success) throw new Error(`Invalid results:\n${v.summarize(result.issues)}`);
+  return result.output;
+}

package/src/text.js

@@ -0,0 +1,9 @@
+const WORDS = ["zero", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine"];
+
+/** Spell out zero through nine and use numerals from 10, as the house style asks. */
+export const num = (n) => (Number.isInteger(n) && n >= 0 && n <= 9 ? WORDS[n] : String(n));
+
+/** "one story", "three stories". */
+export const plural = (n, word, many = `${word}s`) => `${num(n)} ${n === 1 ? word : many}`;
+
+export const cap = (text) => text.charAt(0).toUpperCase() + text.slice(1);