automatica11y 0.0.0-stage → 0.3.0

package/src/plan/mapping.js

@@ -0,0 +1,101 @@
+import { existsSync, readFileSync } from "node:fs";
+import { resolve } from "node:path";
+import { ARCHETYPE_PATTERNS } from "../harness/storybook.js";
+import { ARCHETYPES, parseMappingFile } from "../schema.js";
+
+/** Names that mean the archetype itself, best first. */
+const BASE_NAMES = {
+  button: ["button"],
+  link: ["link", "anchor"],
+  dialog: ["dialog", "modal"],
+  menu: ["menu", "dropdownmenu"],
+  tabs: ["tabs", "tablist"],
+  combobox: ["combobox", "autocomplete", "select"],
+  "form-field": ["input", "textfield", "textinput", "field", "checkbox"],
+  accordion: ["accordion", "collapsible", "disclosure"],
+  tooltip: ["tooltip", "popover"],
+  chart: ["chart", "linechart", "barchart"],
+};
+
+/** Archetypes a template can fill in. Everything else needs a fixture someone writes. */
+export const TEMPLATED = new Set(["button", "link"]);
+
+/** `DialogTrigger` becomes `Dialog Trigger`. `sl-button` becomes `sl button`. */
+const words = (name) => name.replace(/([a-z0-9])([A-Z])/g, "$1 $2").replace(/[-_]/g, " ");
+
+/** How well a name fits an archetype. 0 means it doesn't. */
+function score(archetype, name) {
+  if (!ARCHETYPE_PATTERNS[archetype].test(words(name))) return 0;
+  const compact = name.replace(/[-_\s]/g, "").toLowerCase();
+  const last = words(name).toLowerCase().split(" ").pop();
+  const bases = BASE_NAMES[archetype];
+  if (bases.includes(compact)) return 4;
+  if (bases.includes(last)) return 3;
+  if (bases.some((base) => compact.startsWith(base))) return 2;
+  return 1;
+}
+
+/**
+ * Guess which exports (React) or tags (web components) stand for each archetype.
+ * The result is a starting point. A person or the skill checks it before trusting it.
+ * @param {{ flavor: "react" | "wc", exports?: Array<{ name: string, type: string, parts: string[] }>, tags?: string[] }} input
+ * @returns {Record<string, any>}
+ */
+export function candidateMapping({ flavor, exports = [], tags = [] }) {
+  const names = flavor === "wc" ? tags : exports.filter((e) => /^[A-Z]/.test(e.name)).map((e) => e.name);
+  /** @type {Record<string, any>} */
+  const mapping = {};
+  for (const archetype of ARCHETYPES) {
+    const ranked = names
+      .map((name) => ({ name, score: score(archetype, name) }))
+      .filter((c) => c.score > 0)
+      .sort((a, b) => b.score - a.score || a.name.localeCompare(b.name));
+    if (ranked.length === 0) {
+      mapping[archetype] = { flavor, status: "no-match", candidates: [], fixture: null, reason: `No ${flavor === "wc" ? "custom element" : "export"} looks like the ${archetype} archetype.` };
+      continue;
+    }
+    const best = ranked[0].name;
+    const info = exports.find((e) => e.name === best);
+    const parts = info?.parts ?? [];
+    // Flat compound libraries (DialogRoot, DialogTrigger, DialogContent) have sibling exports that share a prefix.
+    const siblings = flavor === "react" ? names.filter((n) => n !== best && n.startsWith(best) && n.length > best.length) : [];
+    const compound = parts.length > 0 || siblings.length >= 2;
+    const templated = TEMPLATED.has(archetype) && !compound;
+    mapping[archetype] = {
+      flavor,
+      ...(flavor === "wc" ? { tag: best } : { export: best }),
+      status: templated ? "template" : "needs-fixture",
+      candidates: ranked.slice(0, 5).map((c) => c.name),
+      ...(parts.length ? { parts } : siblings.length ? { parts: siblings.slice(0, 12) } : {}),
+      fixture: null,
+      ...(templated ? {} : { reason: compound ? `${best} is built from parts, so a fixture has to assemble them.` : `The ${archetype} archetype needs a fixture someone writes.` }),
+    };
+  }
+  return mapping;
+}
+
+/** Read and validate a `--mapping` file. Throws an Error with a plain message. */
+export function loadMappingFile(path, cwd) {
+  const file = resolve(cwd, path);
+  if (!existsSync(file)) throw new Error(`The mapping file doesn't exist: ${file}`);
+  let json;
+  try {
+    json = JSON.parse(readFileSync(file, "utf8"));
+  } catch (error) {
+    throw new Error(`The mapping file isn't valid JSON: ${error.message}`);
+  }
+  return parseMappingFile(json);
+}
+
+/** Where an authored fixture can sit when the mapping doesn't name one. */
+export function findAuthoredFixture({ cwd, targetId, archetype, mapped }) {
+  if (mapped?.fixture) {
+    const file = resolve(cwd, mapped.fixture);
+    return existsSync(file) ? file : null;
+  }
+  for (const ext of ["jsx", "js"]) {
+    const file = resolve(cwd, "fixtures", targetId, `${archetype}.${ext}`);
+    if (existsSync(file)) return file;
+  }
+  return null;
+}

package/src/plan/resolve-npm.js

@@ -0,0 +1,87 @@
+import { execFile } from "node:child_process";
+
+const FIELDS = ["name", "version", "peerDependencies", "dependencies", "keywords", "customElements", "deprecated"];
+const OTHER_FRAMEWORKS = {
+  vue: "Vue",
+  "@angular/core": "Angular",
+  svelte: "Svelte",
+  "solid-js": "Solid",
+  preact: "Preact",
+  "@builder.io/qwik": "Qwik",
+  "ember-source": "Ember",
+};
+const WEB_COMPONENT_BASES = ["lit", "lit-element", "@lit/reactive-element", "@stencil/core", "@microsoft/fast-element", "@polymer/polymer"];
+
+/**
+ * Read a package's registry metadata without installing it.
+ * @param {string} spec `name`, `name@version`, or `name@range`
+ * @param {{ timeoutMs?: number }} [options]
+ * @returns {Promise<any>} The metadata. Throws an Error with a plain reason when the package can't be read.
+ */
+export function npmView(spec, { timeoutMs = 60_000 } = {}) {
+  return new Promise((resolve, reject) => {
+    execFile(
+      process.platform === "win32" ? "npm.cmd" : "npm",
+      ["view", spec, ...FIELDS, "--json"],
+      { timeout: timeoutMs, maxBuffer: 10 * 1024 * 1024, shell: process.platform === "win32" },
+      (error, stdout, stderr) => {
+        if (error) {
+          const text = `${stdout}\n${stderr}`;
+          if (/E404|404 Not Found|is not in this registry/.test(text)) return reject(new Error(`"${spec}" wasn't found on npm.`));
+          if (/ENOTFOUND|EAI_AGAIN|ECONNREFUSED|ETIMEDOUT|network/i.test(text)) return reject(new Error("The npm registry couldn't be reached."));
+          if ((error).killed) return reject(new Error("The npm registry didn't answer in time."));
+          return reject(new Error(`npm couldn't read "${spec}": ${text.split("\n").find((line) => line.trim())?.trim() ?? error.message}`));
+        }
+        try {
+          const parsed = JSON.parse(stdout);
+          resolve(Array.isArray(parsed) ? parsed[parsed.length - 1] : parsed);
+        } catch {
+          reject(new Error(`npm returned something unreadable for "${spec}".`));
+        }
+      },
+    );
+  });
+}
+
+/**
+ * Guess how a package renders from its metadata alone. React wins when both signals appear.
+ * `npm` means the metadata can't say, so the run decides after it installs and loads the package.
+ * @param {any} meta
+ * @returns {{ kind: "npm-react" | "npm-wc" | "npm-unsupported" | "npm", framework: string | null, reason: string }}
+ */
+export function detectFlavor(meta) {
+  const peers = meta.peerDependencies ?? {};
+  const deps = meta.dependencies ?? {};
+  if ("react" in peers || "react-dom" in peers || "react" in deps) {
+    return { kind: "npm-react", framework: "React", reason: "The package lists react as a dependency." };
+  }
+  if (meta.customElements) return { kind: "npm-wc", framework: "Web components", reason: "The package has a customElements manifest." };
+  for (const [name, label] of Object.entries(OTHER_FRAMEWORKS)) {
+    if (name in peers) return { kind: "npm-unsupported", framework: label, reason: `The package needs ${label}.` };
+  }
+  const base = WEB_COMPONENT_BASES.find((name) => name in deps || name in peers);
+  if (base) return { kind: "npm-wc", framework: "Web components", reason: `The package builds on ${base}.` };
+  return { kind: "npm", framework: null, reason: "The metadata doesn't say. The run decides after it loads the package." };
+}
+
+/**
+ * Fill in a classified npm target: the concrete version and the framework guess.
+ * @param {import("./classify.js").ClassifiedTarget} target
+ * @param {(spec: string) => Promise<any>} view
+ * @returns {Promise<import("./classify.js").ClassifiedTarget>}
+ */
+export async function resolveNpmTarget(target, view) {
+  if (target.status !== "ok" || target.kind !== "npm" || !target.resolved) return target;
+  const { name, requested } = target.resolved;
+  try {
+    const meta = await view(`${name}@${requested ?? "latest"}`);
+    const flavor = detectFlavor(meta);
+    return {
+      ...target,
+      kind: /** @type {any} */ (flavor.kind),
+      resolved: { ...target.resolved, version: meta.version ?? null, framework: flavor.framework, detectedBy: flavor.reason },
+    };
+  } catch (error) {
+    return { ...target, status: "failed", reason: error instanceof Error ? error.message : String(error), kind: null, evidenceLevel: null, resolved: null };
+  }
+}

package/src/report/comparison.js

@@ -0,0 +1,183 @@
+import { ARCHETYPES } from "../schema.js";
+import { num, plural } from "../text.js";
+import {
+  ENGINE_NAMES,
+  FINDINGS_NOTE,
+  TIER_NAMES,
+  cell,
+  code,
+  configLabel,
+  finish,
+  notTestableLines,
+  reportFooter,
+  reportHeader,
+  sentence,
+  targetSection,
+} from "./parts.js";
+
+const TIER_ORDER = ["rules", "interactions", "vsr"];
+
+/** The archetype rows: component archetypes in a fixed order, then whole pages, then Storybook stories. */
+function archetypeKeys(results) {
+  const keys = new Set();
+  for (const target of results.targets) {
+    for (const key of Object.keys(target.archetypes)) if (!key.startsWith("story:")) keys.add(key);
+    for (const key of Object.keys(target.storybook?.archetypeMatches ?? {})) keys.add(key);
+  }
+  return [...ARCHETYPES.filter((a) => keys.has(a)), ...[...keys].filter((k) => !ARCHETYPES.includes(k) && k !== "page"), ...(keys.has("page") ? ["page"] : [])];
+}
+
+/** Configs for one target and one archetype row, as labeled lines for the findings table. */
+function rowsFor(planTarget, target, key) {
+  // A target that failed outright has nothing per archetype. One whose archetypes were all gaps still says what each gap was.
+  if (target.status !== "ran" && Object.keys(target.archetypes).length === 0) return [{ label: "-", note: `${target.status}: ${sentence(target.reason ?? "")}` }];
+  if (target.storybook) {
+    const ids = target.storybook.archetypeMatches[key] ?? [];
+    const stories = ids.map((id) => target.archetypes[`story:${id}`]).filter(Boolean);
+    if (key === "page") return [{ label: "-", note: "not-applicable: component evidence" }];
+    if (stories.length === 0) return [{ label: "-", note: "gap: no story matched this archetype" }];
+    const configs = stories.filter((s) => s.status === "ran").map((s) => s.configs[0]);
+    if (configs.length === 0) return [{ label: `${plural(stories.length, "story", "stories")}`, note: "gap: none of the matching stories rendered" }];
+    return [{ label: `${plural(configs.length, "story", "stories")} of ${num(stories.length)}`, configs, always: true }];
+  }
+  const archetype = target.archetypes[key];
+  if (!archetype) return [{ label: "-", note: key === "page" ? "not-applicable: component evidence" : "not-applicable: page evidence" }];
+  if (archetype.status === "gap") return [{ label: "-", note: `gap: ${sentence(archetype.reason ?? "no fixture")}` }];
+  return archetype.configs.map((config) => ({ label: configLabel(config).replace(/^ \(|\)$/g, "") || "-", configs: [config] }));
+}
+
+/** A coverage cell for one tier: what happened to that tier for the target and archetype. */
+function tierStatus(rows, tier) {
+  const row = rows[0];
+  if (row.note) return row.note.split(":")[0];
+  const statuses = rows.flatMap((r) => r.configs.map((c) => c.tiers[tier]?.status).filter(Boolean));
+  if (statuses.length === 0) return "-";
+  if (statuses.every((s) => s === statuses[0])) return statuses[0];
+  return statuses.includes("ran") ? "partly ran" : statuses[0];
+}
+
+function coverageTable(plan, results, tier, keys) {
+  const head = ["Archetype", ...results.targets.map((t) => t.id)];
+  const lines = [`| ${head.join(" | ")} |`, `| ${head.map(() => "---").join(" | ")} |`];
+  for (const key of keys) {
+    const cells = results.targets.map((target) => {
+      const planTarget = plan.targets.find((t) => t.id === target.id);
+      const rows = rowsFor(planTarget, target, key);
+      const status = tierStatus(rows, tier);
+      // Name the configuration only when there's more than one to tell apart, or the row stands for several stories.
+      const withTier = rows.filter((r) => r.configs?.some((c) => c.tiers[tier]));
+      const labels = withTier.length > 1 || withTier[0]?.always ? withTier.map((r) => r.label).filter((l) => l && l !== "-") : [];
+      return labels.length ? `${status} (${labels.join("; ")})` : status;
+    });
+    lines.push(`| ${key} | ${cells.join(" | ")} |`);
+  }
+  return lines;
+}
+
+/** Rule IDs for one engine across configs, with element counts, for a findings cell. */
+function ruleCell(configs, engine) {
+  const found = new Map();
+  let review = 0;
+  let statuses = [];
+  for (const config of configs) {
+    const result = config.tiers.rules?.engines?.[engine];
+    if (!result) continue;
+    statuses.push(result.status);
+    if (result.status !== "ran") continue;
+    for (const finding of result.violations) found.set(finding.ruleId, (found.get(finding.ruleId) ?? 0) + finding.nodeCount);
+    review += result.incomplete.length;
+  }
+  if (statuses.length === 0) return "-";
+  if (!statuses.includes("ran")) return statuses[0];
+  const list = [...found.entries()].sort(([a], [b]) => a.localeCompare(b));
+  const shown = list.slice(0, 4).map(([id, n]) => `${code(id)} (${num(n)})`).join(", ");
+  const text = list.length ? `${shown}${list.length > 4 ? `, and ${num(list.length - 4)} more` : ""}` : "none found";
+  return `${text}${review ? `; ${num(review)} to review` : ""}`;
+}
+
+function interactionsCell(configs) {
+  const checks = configs.flatMap((c) => c.tiers.interactions?.checks ?? []);
+  if (checks.length === 0) return configs.map((c) => c.tiers.interactions?.status).find(Boolean) ?? "-";
+  const failed = checks.filter((c) => c.result === "fail").map((c) => code(c.name));
+  const errors = checks.filter((c) => c.result === "error").length;
+  const parts = [failed.length ? `failed: ${failed.slice(0, 3).join(", ")}${failed.length > 3 ? `, and ${num(failed.length - 3)} more` : ""}` : "no failures", errors ? `${plural(errors, "error")}` : null];
+  return parts.filter(Boolean).join("; ");
+}
+
+function vsrCell(configs) {
+  const walks = configs.map((c) => c.tiers.vsr).filter((v) => v?.status === "ran");
+  if (walks.length === 0) return configs.map((c) => c.tiers.vsr?.status).find(Boolean) ?? "-";
+  const flags = walks.flatMap((v) => v.flags.map((f) => f.phrase));
+  const unique = [...new Set(flags)];
+  return flags.length ? `${plural(flags.length, "phrase")} flagged: ${unique.slice(0, 3).map(code).join(", ")}` : "none flagged";
+}
+
+function findingsTable(plan, results, key) {
+  const head = ["Target", "Configuration", ENGINE_NAMES.axe, ENGINE_NAMES.ibm, "Interactions", "Virtual screen reader (simulated)"];
+  const lines = [`| ${head.join(" | ")} |`, `| ${head.map(() => "---").join(" | ")} |`];
+  for (const target of results.targets) {
+    const planTarget = plan.targets.find((t) => t.id === target.id);
+    for (const row of rowsFor(planTarget, target, key)) {
+      if (row.note) {
+        lines.push(`| ${cell(target.id)} | ${cell(row.label)} | ${cell(row.note)} | | | |`);
+        continue;
+      }
+      const c = row.configs;
+      lines.push(`| ${cell(target.id)} | ${cell(row.label)} | ${cell(ruleCell(c, "axe"))} | ${cell(ruleCell(c, "ibm"))} | ${cell(interactionsCell(c))} | ${cell(vsrCell(c))} |`);
+    }
+  }
+  return lines;
+}
+
+/** Violations by axe impact and by IBM Toolkit level, in separate tables. Counts are never added across engines. */
+function impactTables(results, engines) {
+  const lines = [];
+  if (engines.includes("axe")) {
+    lines.push("### axe-core, violations by impact.", "", "| Target | Critical | Serious | Moderate | Minor | Needs review |", "| --- | --- | --- | --- | --- | --- |");
+    for (const target of results.targets) {
+      const s = target.summary.engines.axe;
+      lines.push(s?.status === "ran" ? `| ${target.id} | ${["critical", "serious", "moderate", "minor"].map((k) => s.violationsByImpact[k]).join(" | ")} | ${s.needsReview} |` : `| ${target.id} | ${s?.status ?? target.status} | | | | |`);
+    }
+    lines.push("");
+  }
+  if (engines.includes("ibm")) {
+    lines.push("### IBM Equal Access, violations by Toolkit level.", "", "| Target | Level 1 | Level 2 | Level 3 | Level 4 | Needs review |", "| --- | --- | --- | --- | --- | --- |");
+    for (const target of results.targets) {
+      const s = target.summary.engines.ibm;
+      lines.push(s?.status === "ran" ? `| ${target.id} | ${["1", "2", "3", "4"].map((k) => s.violationsByToolkitLevel[k]).join(" | ")} | ${s.needsReview} |` : `| ${target.id} | ${s?.status ?? target.status} | | | | |`);
+    }
+    lines.push("");
+  }
+  return lines;
+}
+
+/**
+ * Render report.md for a comparison of two or more targets.
+ * Every target ran with the same settings, so the columns line up. A gap or a failure shows in its cell, and never reads as a pass.
+ * @param {{ plan: any, results: any }} input
+ */
+export function renderComparison({ plan, results }) {
+  const o = plan.options;
+  const lines = reportHeader(plan, results, "Accessibility comparison.");
+  lines.push(
+    `Every target was checked with the same settings: WCAG ${o.wcag}, level ${o.level}, ${o.engines.map((e) => ENGINE_NAMES[e]).join(" and ")}, and the same archetypes${o.archetypes ? ` (${o.archetypes.join(", ")})` : ""}. A difference below comes from the targets and not from the settings.`,
+    "",
+  );
+  const keys = archetypeKeys(results);
+
+  lines.push("## Coverage.", "", "Each cell says what happened to that tier for that target and archetype: ran, gap, not-testable, not-applicable, skipped, or failed. A gap, a failure, or a target that wasn't testable is a finding. It never counts as a pass.", "");
+  for (const tier of TIER_ORDER.filter((t) => o.tiers.includes(t))) {
+    lines.push(`### ${TIER_NAMES[tier]}${tier === "vsr" ? " (simulated)" : ""}.`, "", ...coverageTable(plan, results, tier, keys), "");
+  }
+  lines.push(...notTestableLines(results));
+
+  lines.push("## Findings.", "", FINDINGS_NOTE, "", ...(o.tiers.includes("rules") ? impactTables(results, o.engines) : []));
+  for (const key of keys) {
+    lines.push(`### ${key === "page" ? "Whole pages" : key}.`, "", ...findingsTable(plan, results, key), "");
+  }
+
+  lines.push("## Details by target.", "", "Every finding, with its elements, rule help, and logs, for each target in turn.", "");
+  for (const target of results.targets) lines.push(targetSection(plan.targets.find((t) => t.id === target.id), target));
+  lines.push(...reportFooter(results));
+  return finish(lines);
+}

package/src/report/index.js

@@ -0,0 +1,10 @@
+import { renderComparison } from "./comparison.js";
+import { renderSingleReport } from "./single.js";
+
+/**
+ * Render report.md. A comparison of two or more targets gets the side-by-side report. Anything else gets the single report.
+ * @param {{ plan: any, results: any }} input
+ */
+export function renderReport(input) {
+  return input.plan.command === "compare" && input.results.targets.length > 1 ? renderComparison(input) : renderSingleReport(input);
+}