automatica11y 0.0.0-stage → 0.2.0

package/src/harness/bundle.js

@@ -0,0 +1,49 @@
+import { existsSync, mkdirSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+
+const ASSET_LOADERS = /** @type {Record<string, import("esbuild").Loader>} */ (Object.fromEntries(
+  [".png", ".jpg", ".jpeg", ".gif", ".svg", ".webp", ".woff", ".woff2", ".ttf", ".eot"].map((ext) => [ext, "dataurl"]),
+));
+
+/**
+ * Bundle fixture entries into one folder of browser-ready ES modules, plus one HTML shell per entry.
+ * esbuild loads here and only here. It doesn't type-check, and fixtures don't need it to.
+ * @param {{ entries: Record<string, string>, outdir: string, workDir: string, react?: boolean }} options
+ * @returns {Promise<Record<string, string>>} Entry name to page path, for example `{ dialog: "/dialog.html" }`.
+ */
+export async function bundleEntries({ entries, outdir, workDir, react = false }) {
+  const esbuild = await import("esbuild");
+  mkdirSync(outdir, { recursive: true });
+  try {
+    await esbuild.build({
+      entryPoints: entries,
+      outdir,
+      bundle: true,
+      format: "esm",
+      platform: "browser",
+      target: "es2022",
+      jsx: "automatic",
+      absWorkingDir: workDir,
+      nodePaths: [join(workDir, "node_modules")],
+      // One copy of React for the library and the fixture, or hooks break.
+      alias: react ? { react: join(workDir, "node_modules", "react"), "react-dom": join(workDir, "node_modules", "react-dom") } : {},
+      loader: ASSET_LOADERS,
+      define: { "process.env.NODE_ENV": '"development"' },
+      logLevel: "silent",
+    });
+  } catch (error) {
+    const messages = (error.errors ?? []).slice(0, 3).map((e) => `${e.text}${e.location ? ` (${e.location.file}:${e.location.line})` : ""}`);
+    throw new Error(`Bundling failed: ${messages.join("; ") || error.message}`);
+  }
+  /** @type {Record<string, string>} */
+  const pages = {};
+  for (const name of Object.keys(entries)) {
+    const css = existsSync(join(outdir, `${name}.css`)) ? `<link rel="stylesheet" href="./${name}.css">` : "";
+    writeFileSync(
+      join(outdir, `${name}.html`),
+      `<!doctype html>\n<html lang="en">\n<head><meta charset="utf-8"><title>${name}</title><link rel="icon" href="data:,">${css}</head>\n<body><div id="root"></div><script type="module" src="./${name}.js"></script></body>\n</html>\n`,
+    );
+    pages[name] = `/${name}.html`;
+  }
+  return pages;
+}

package/src/harness/npm-install.js

@@ -0,0 +1,65 @@
+import { execFile } from "node:child_process";
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+
+const NPM_FLAGS = ["--ignore-scripts", "--no-audit", "--no-fund", "--prefer-offline", "--loglevel=error"];
+
+/** Run npm in `cwd`. Resolves with stdout and rejects with npm's own words. */
+export function runNpm(args, cwd, { timeoutMs = 300_000 } = {}) {
+  return new Promise((resolve, reject) => {
+    execFile(
+      process.platform === "win32" ? "npm.cmd" : "npm",
+      args,
+      { cwd, timeout: timeoutMs, maxBuffer: 20 * 1024 * 1024, shell: process.platform === "win32" },
+      (error, stdout, stderr) => {
+        if (error) reject(Object.assign(new Error(`${stderr}\n${stdout}`.trim() || error.message), { code: "NPM_FAILED" }));
+        else resolve(stdout);
+      },
+    );
+  });
+}
+
+/** The version of an installed package, or null. */
+export function installedVersion(dir, name) {
+  try {
+    return JSON.parse(readFileSync(join(dir, "node_modules", name, "package.json"), "utf8")).version ?? null;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Install a package into its own directory, never next to another target's install.
+ * npm adds the peer dependencies. A React package also needs react-dom, so add it when the peers left it out.
+ * Install scripts stay off, because the code is untrusted until it runs in the browser sandbox.
+ * @param {{ dir: string, name: string, version: string, flavor: "react" | "wc" | "unknown", run?: typeof runNpm }} options
+ */
+export async function installPackage({ dir, name, version, flavor, run = runNpm }) {
+  mkdirSync(dir, { recursive: true });
+  if (!existsSync(join(dir, "package.json"))) writeFileSync(join(dir, "package.json"), JSON.stringify({ name: "automatica11y-target", private: true }));
+  /** @type {string[]} */
+  const warnings = [];
+  try {
+    await run(["install", `${name}@${version}`, ...NPM_FLAGS], dir);
+  } catch (error) {
+    if (!/ERESOLVE|peer dep/i.test(String(error.message))) throw new Error(`npm couldn't install ${name}@${version}: ${firstLine(error.message)}`);
+    await run(["install", `${name}@${version}`, "--legacy-peer-deps", ...NPM_FLAGS], dir).catch((retry) => {
+      throw new Error(`npm couldn't install ${name}@${version}: ${firstLine(retry.message)}`);
+    });
+    warnings.push(`npm couldn't satisfy ${name}'s peer dependencies, so it installed them loosely (--legacy-peer-deps). Results may not match a supported setup.`);
+  }
+  if (flavor === "react") {
+    const react = installedVersion(dir, "react");
+    if (!react) {
+      await run(["install", "react", "react-dom", ...NPM_FLAGS, "--legacy-peer-deps"], dir);
+      warnings.push(`${name} didn't bring in react, so the latest react and react-dom were added.`);
+    } else if (!installedVersion(dir, "react-dom")) {
+      await run(["install", `react-dom@${react}`, ...NPM_FLAGS, "--legacy-peer-deps"], dir);
+    }
+  }
+  return { dir, warnings, react: installedVersion(dir, "react"), reactDom: installedVersion(dir, "react-dom"), version: installedVersion(dir, name) };
+}
+
+function firstLine(text) {
+  return String(text).split("\n").find((line) => line.trim())?.replace(/^npm (error|ERR!)\s*/i, "").trim() ?? "unknown error";
+}

package/src/harness/npm-react.js

@@ -0,0 +1,39 @@
+/** React flavor: the entry files and button and link templates that go into a bundle. */
+
+/** Mounts a fixture's default export. */
+export const entry = (fixturePath, _pkg) => `import { createElement } from "react";
+import { createRoot } from "react-dom/client";
+import Fixture from ${JSON.stringify(fixturePath)};
+const libA11y = new URLSearchParams(location.search).get("libA11y") === "on";
+createRoot(document.getElementById("root")).render(createElement(Fixture, { libA11y }));
+`;
+
+/** Loads the whole package, lists its exports, and records compound parts such as Dialog.Trigger. */
+export const discoverEntry = (pkg) => `import * as lib from ${JSON.stringify(pkg)};
+const out = [];
+for (const name of Object.keys(lib)) {
+  const value = lib[name];
+  const type = typeof value;
+  if (value === null || (type !== "function" && type !== "object")) continue;
+  const parts = Object.keys(value).filter((key) => /^[A-Z]/.test(key)).slice(0, 30);
+  out.push({ name, type, parts });
+}
+window.__a11yExports = out;
+`;
+
+/**
+ * A fixture for the simple archetypes, from the export name alone.
+ * Compound components (dialog, tabs, menu) can't be guessed, so they need a fixture someone writes.
+ */
+export function template(archetype, pkg, exportName) {
+  const body = {
+    button: `<Component data-a11y-trigger data-a11y-root type="button">Save</Component>`,
+    link: `<Component data-a11y-trigger data-a11y-root href="#top">Read more</Component>`,
+  }[archetype];
+  if (!body) return null;
+  return `import { ${exportName} as Component } from ${JSON.stringify(pkg)};
+export default function Fixture() {
+  return ${body};
+}
+`;
+}

package/src/harness/npm-wc.js

@@ -0,0 +1,30 @@
+/** Web component flavor: framework-free entries and templates. */
+
+/** Loads the package so its custom elements get defined, then calls the fixture's default export, `mount(container)`. */
+export const entry = (fixturePath, pkg) => `import ${JSON.stringify(pkg)};
+import mount from ${JSON.stringify(fixturePath)};
+const libA11y = new URLSearchParams(location.search).get("libA11y") === "on";
+await mount(document.getElementById("root"), { libA11y });
+`;
+
+/** Loads the whole package. The page's init script records every custom element the package defines. */
+export const discoverEntry = (pkg) => `import * as lib from ${JSON.stringify(pkg)};
+window.__a11yExports = Object.keys(lib).map((name) => ({ name, type: typeof lib[name], parts: [] }));
+`;
+
+/** A fixture for the simple archetypes, from the tag name alone. */
+export function template(archetype, tag) {
+  const attrs = {
+    button: `el.textContent = "Save";`,
+    link: `el.setAttribute("href", "#top");\n  el.textContent = "Read more";`,
+  }[archetype];
+  if (!attrs) return null;
+  return `export default function mount(container) {
+  const el = document.createElement(${JSON.stringify(tag)});
+  el.setAttribute("data-a11y-trigger", "");
+  el.setAttribute("data-a11y-root", "");
+  ${attrs}
+  container.append(el);
+}
+`;
+}

package/src/harness/shadow.js

@@ -0,0 +1,42 @@
+/**
+ * Runs before any page script. It notes every closed shadow root, because axe and IBM can't see inside one
+ * and would report nothing, which reads as clean.
+ */
+export function recordClosedShadowRoots() {
+  const original = Element.prototype.attachShadow;
+  const hosts = [];
+  Object.defineProperty(window, "__a11yClosedShadowHosts", { value: hosts, configurable: true });
+  Element.prototype.attachShadow = function attachShadow(init) {
+    if (init && init.mode === "closed") hosts.push(this.localName);
+    return original.call(this, init);
+  };
+}
+
+/**
+ * The closed shadow roots the page created, as host tag names with counts, for example `{ "x-thing": 2 }`.
+ * @param {import("playwright-core").Page} page
+ * @returns {Promise<Record<string, number>>}
+ */
+export async function closedShadowHosts(page) {
+  const hosts = await page.evaluate(() => /** @type {any} */ (window).__a11yClosedShadowHosts ?? []).catch(() => []);
+  /** @type {Record<string, number>} */
+  const counts = {};
+  for (const host of hosts) counts[host] = (counts[host] ?? 0) + 1;
+  return counts;
+}
+
+/** A readable not-testable entry for each host tag. */
+export function notTestableEntries(counts) {
+  return Object.entries(counts).map(([tag, n]) => `closed shadow root in <${tag}> (${n}): its content isn't visible to the rule engines`);
+}
+
+/** Runs before any page script. Notes every custom element the page or a package defines. */
+export function recordCustomElements() {
+  const original = customElements.define.bind(customElements);
+  const tags = [];
+  Object.defineProperty(window, "__a11yDefined", { value: tags, configurable: true });
+  customElements.define = (name, constructor, options) => {
+    tags.push(name);
+    return original(name, constructor, options);
+  };
+}

package/src/harness/static-serve.js

@@ -0,0 +1,68 @@
+import { createReadStream, existsSync, statSync } from "node:fs";
+import { createServer } from "node:http";
+import { extname, join, normalize, resolve, sep } from "node:path";
+
+const TYPES = {
+  ".html": "text/html; charset=utf-8",
+  ".htm": "text/html; charset=utf-8",
+  ".js": "text/javascript; charset=utf-8",
+  ".mjs": "text/javascript; charset=utf-8",
+  ".css": "text/css; charset=utf-8",
+  ".json": "application/json; charset=utf-8",
+  ".svg": "image/svg+xml",
+  ".png": "image/png",
+  ".jpg": "image/jpeg",
+  ".jpeg": "image/jpeg",
+  ".gif": "image/gif",
+  ".webp": "image/webp",
+  ".ico": "image/x-icon",
+  ".woff": "font/woff",
+  ".woff2": "font/woff2",
+  ".ttf": "font/ttf",
+  ".txt": "text/plain; charset=utf-8",
+  ".map": "application/json",
+  ".wasm": "application/wasm",
+};
+
+/**
+ * Serve a directory over http://127.0.0.1 so module scripts and relative assets work. Never use file://.
+ * Binds a free port, serves only files inside `root`, and lists no directories.
+ * @param {string} root
+ * @returns {Promise<{ origin: string, close: () => Promise<void> }>}
+ */
+export async function serveStatic(root) {
+  const base = resolve(root);
+  const server = createServer((request, response) => {
+    try {
+      const pathname = decodeURIComponent(new URL(request.url ?? "/", "http://localhost").pathname);
+      let file = resolve(join(base, normalize(pathname)));
+      if (file !== base && !file.startsWith(base + sep)) {
+        response.writeHead(403).end("Forbidden");
+        return;
+      }
+      if (existsSync(file) && statSync(file).isDirectory()) file = join(file, "index.html");
+      if (!existsSync(file) || !statSync(file).isFile()) {
+        response.writeHead(404, { "content-type": "text/plain" }).end("Not found");
+        return;
+      }
+      response.writeHead(200, { "content-type": TYPES[extname(file).toLowerCase()] ?? "application/octet-stream", "cache-control": "no-store" });
+      if (request.method === "HEAD") response.end();
+      else createReadStream(file).pipe(response);
+    } catch {
+      response.writeHead(400).end("Bad request");
+    }
+  });
+  await new Promise((done, fail) => {
+    server.once("error", fail);
+    server.listen(0, "127.0.0.1", () => done(undefined));
+  });
+  const address = /** @type {import("node:net").AddressInfo} */ (server.address());
+  return {
+    origin: `http://127.0.0.1:${address.port}`,
+    close: () =>
+      new Promise((done) => {
+        server.closeAllConnections?.();
+        server.close(() => done());
+      }),
+  };
+}

package/src/harness/storybook.js

@@ -0,0 +1,116 @@
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+
+/** Name patterns that suggest an archetype. They read the story title, name, and tags. */
+export const ARCHETYPE_PATTERNS = {
+  button: /\bbuttons?\b/i,
+  link: /\blinks?\b|\banchors?\b/i,
+  dialog: /\bdialogs?\b|\bmodals?\b|\bdrawers?\b|\balert ?dialogs?\b/i,
+  menu: /\bmenus?\b|\bdropdown ?menus?\b/i,
+  tabs: /\btabs?\b|\btab ?list\b/i,
+  combobox: /\bcombo ?box(es)?\b|\bautocomplete\b|\btypeahead\b|\bselect\b/i,
+  "form-field": /\binputs?\b|\btext ?(field|area|box)\b|\bform\b|\bcheckbox(es)?\b|\bradio\b|\bswitch\b|\bfield\b/i,
+  accordion: /\baccordions?\b|\bcollaps(e|ible)\b|\bdisclosure\b/i,
+  tooltip: /\btooltips?\b|\bpopovers?\b/i,
+  chart: /\bcharts?\b|\bgraphs?\b|\bplots?\b/i,
+};
+
+/**
+ * Pull the stories out of a Storybook index. Handles `index.json` (`entries`, with docs pages) and the older `stories.json`.
+ * @param {any} index
+ * @returns {Array<{ id: string, title: string, name: string, tags: string[] }>}
+ */
+export function listStories(index) {
+  const table = index.entries ?? index.stories ?? {};
+  return Object.values(table)
+    .filter((entry) => entry && typeof entry === "object" && (entry.type === undefined || entry.type === "story"))
+    .map((entry) => ({
+      id: String(entry.id),
+      title: String(entry.title ?? entry.kind ?? ""),
+      name: String(entry.name ?? entry.story ?? ""),
+      tags: Array.isArray(entry.tags) ? entry.tags.map(String) : [],
+    }))
+    .sort((a, b) => a.id.localeCompare(b.id));
+}
+
+/** The archetypes a story's title, name, and tags suggest. */
+export function matchArchetypes(story) {
+  const text = `${story.title} ${story.name} ${story.tags.join(" ")}`;
+  return Object.entries(ARCHETYPE_PATTERNS)
+    .filter(([, pattern]) => pattern.test(text))
+    .map(([archetype]) => archetype);
+}
+
+/**
+ * Choose which stories to audit. With `archetypes`, keep only stories that match one. Then cap the count.
+ * The cap spreads over components, taking one story per title in turn, so one big component can't use up the whole budget.
+ * The order is stable, so two runs pick the same stories.
+ * @param {ReturnType<typeof listStories>} stories
+ * @param {{ archetypes?: string[] | null, max: number }} options
+ */
+export function selectStories(stories, { archetypes = null, max }) {
+  /** Every archetype each story suggests. The comparison matrix uses this even when no filter is set. */
+  /** @type {Record<string, string[]>} */
+  const all = {};
+  for (const story of stories) for (const a of matchArchetypes(story)) (all[a] ??= []).push(story.id);
+  const filtered = Boolean(archetypes?.length);
+  let pool = stories;
+  /** @type {Record<string, string[]>} */
+  let matchedByArchetype = all;
+  if (filtered) {
+    matchedByArchetype = {};
+    pool = stories.filter((story) => {
+      const hits = matchArchetypes(story).filter((a) => archetypes.includes(a));
+      for (const a of hits) (matchedByArchetype[a] ??= []).push(story.id);
+      return hits.length > 0;
+    });
+  }
+  const byTitle = new Map();
+  for (const story of pool) byTitle.set(story.title, [...(byTitle.get(story.title) ?? []), story]);
+  const titles = [...byTitle.keys()].sort();
+  const selected = [];
+  for (let round = 0; selected.length < Math.min(max, pool.length); round += 1) {
+    for (const title of titles) {
+      const story = byTitle.get(title)[round];
+      if (story && selected.length < max) selected.push(story);
+    }
+  }
+  selected.sort((a, b) => a.id.localeCompare(b.id));
+  return { selected, total: stories.length, matched: pool.length, truncated: pool.length > selected.length, matchedByArchetype, filtered };
+}
+
+/** The URL that renders one story on its own. `base` must end with a slash. */
+export function storyUrl(base, id) {
+  return `${base}iframe.html?id=${encodeURIComponent(id)}&viewMode=story`;
+}
+
+/**
+ * Read a Storybook's index from disk or over HTTP.
+ * @param {{ path?: string | null, url?: string | null, index: string }} resolved
+ * @param {Function} [doFetch]
+ */
+export async function readIndex(resolved, doFetch = globalThis.fetch) {
+  if (resolved.path) return JSON.parse(readFileSync(join(resolved.path, resolved.index), "utf8"));
+  const response = await doFetch(new URL(resolved.index, /** @type {string} */ (resolved.url)), { signal: AbortSignal.timeout(15000) });
+  if (!response.ok) throw new Error(`The Storybook index responded with HTTP ${response.status}.`);
+  return response.json();
+}
+
+/**
+ * Wait until Storybook says a story rendered, or failed. Storybook marks `body` with a class for each outcome.
+ * @param {import("playwright-core").Page} page
+ */
+export async function waitForStory(page, timeout = 20_000) {
+  await page.waitForFunction(
+    () => ["sb-show-main", "sb-show-errordisplay", "sb-show-nopreview"].some((name) => document.body.classList.contains(name)),
+    undefined,
+    { timeout },
+  );
+  const outcome = await page.evaluate(() => ({
+    error: document.body.classList.contains("sb-show-errordisplay"),
+    missing: document.body.classList.contains("sb-show-nopreview"),
+    detail: document.querySelector(".sb-errordisplay_main, #error-message")?.textContent?.trim().slice(0, 200) ?? "",
+  }));
+  if (outcome.error) throw new Error(`The story threw an error${outcome.detail ? `: ${outcome.detail}` : "."}`);
+  if (outcome.missing) throw new Error("Storybook couldn't find this story.");
+}

package/src/harness/url.js

@@ -0,0 +1,41 @@
+import { VIEWPORT } from "./browser.js";
+import { recordClosedShadowRoots, recordCustomElements } from "./shadow.js";
+
+const NETWORK_IDLE_MS = 30_000;
+
+/**
+ * Open a page in its own browser context and wait for the network to go quiet.
+ * The returned `warnings` hold anything the report should mention, such as a network that never went idle.
+ * @param {import("playwright-core").Browser} browser
+ * @param {string} url
+ * @param {{ viewport?: { width: number, height: number }, forcedColors?: boolean, beforeGoto?: (page: import("playwright-core").Page) => void | Promise<unknown>, waitUntil?: "load" | "networkidle" }} [options]
+ *   `beforeGoto` runs before navigation, so a caller can attach console listeners that see the first messages.
+ *   `waitUntil` defaults to `networkidle`. Local fixture pages don't need to wait for the network.
+ */
+export async function openPage(browser, url, { viewport = VIEWPORT, forcedColors = false, beforeGoto, waitUntil = "networkidle" } = {}) {
+  const context = await browser.newContext({ viewport, deviceScaleFactor: 1, forcedColors: forcedColors ? "active" : "none" });
+  await context.addInitScript(recordClosedShadowRoots);
+  await context.addInitScript(recordCustomElements);
+  /** @type {string[]} */
+  const warnings = [];
+  try {
+    const page = await context.newPage();
+    page.setDefaultTimeout(30_000);
+    await beforeGoto?.(page);
+    let response;
+    try {
+      response = await page.goto(url, { waitUntil, timeout: NETWORK_IDLE_MS });
+    } catch (error) {
+      if (error instanceof Error && error.name === "TimeoutError" && page.url() !== "about:blank") {
+        warnings.push(`The network never went idle within ${NETWORK_IDLE_MS / 1000} seconds. The page was checked as it stood.`);
+      } else {
+        throw error;
+      }
+    }
+    if (response && response.status() >= 400) throw new Error(`The page responded with HTTP ${response.status()}.`);
+    return { context, page, warnings, close: () => context.close().catch(() => {}) };
+  } catch (error) {
+    await context.close().catch(() => {});
+    throw error;
+  }
+}

package/src/plan/build-plan.js

@@ -0,0 +1,62 @@
+import { classifyTarget, parseTargetInput } from "./classify.js";
+import { npmView, resolveNpmTarget } from "./resolve-npm.js";
+import { readToolVersions } from "../env/versions.js";
+
+/** @param {string} text */
+function slug(text) {
+  const out = text.toLowerCase().replace(/^@/, "").replace(/[^a-z0-9._-]+/g, "-").replace(/^-+|-+$/g, "");
+  return out || "target";
+}
+
+/**
+ * Classify every target and assemble plan.json. Nothing here installs a package or launches a browser.
+ * @param {{
+ *   command: "audit" | "compare",
+ *   targets: string[],
+ *   options: import("valibot").InferOutput<typeof import("../schema.js").PlanSchema>["options"],
+ *   browserVersion?: string | null,
+ *   ctx?: import("./classify.js").ClassifyContext,
+ *   now?: Date,
+ * }} input
+ */
+export async function buildPlan({ command, targets, options, browserVersion = null, ctx = {}, now = new Date() }) {
+  const view = ctx.npmView ?? npmView;
+  const classified = await Promise.all(targets.map(async (raw) => resolveNpmTarget(await classifyTarget(raw, ctx), view)));
+  const seen = new Map();
+  const planTargets = classified.map((target) => {
+    const base = slug(target.label ?? target.name);
+    const count = (seen.get(base) ?? 0) + 1;
+    seen.set(base, count);
+    return {
+      id: count === 1 ? base : `${base}-${count}`,
+      input: target.input,
+      label: target.label ?? target.name,
+      status: target.status,
+      reason: target.reason,
+      kind: target.kind,
+      evidenceLevel: target.evidenceLevel,
+      resolved: target.resolved,
+      mapping: null,
+    };
+  });
+  return {
+    schema: /** @type {const} */ (1),
+    createdAt: now.toISOString(),
+    command,
+    options,
+    tools: readToolVersions({ chromium: browserVersion }),
+    targets: planTargets,
+  };
+}
+
+/** Labels the user typed must be unique. Returns the first duplicate, or null. */
+export function findDuplicateLabel(rawTargets) {
+  const seen = new Set();
+  for (const raw of rawTargets) {
+    const { label } = parseTargetInput(raw.trim());
+    if (!label) continue;
+    if (seen.has(label)) return label;
+    seen.add(label);
+  }
+  return null;
+}