@wcag-audit/cli 1.0.0-alpha.3

package/src/checkers/consistency.js

@@ -0,0 +1,222 @@
+// Multi-page consistency checker. Covers:
+//
+//   3.2.3 Consistent Navigation     — repeated navigation in same relative order
+//   3.2.4 Consistent Identification — components with same function get same name
+//   3.2.6 Consistent Help           — help mechanism in the same place across pages
+//
+// Strategy: starting from the audited URL, follow up to N internal links
+// (same origin), extract a normalized "navigation fingerprint" and a
+// "components fingerprint" from each page, then compare. Differences in
+// link order, component naming, or help-link position become findings.
+
+import { URL } from "node:url";
+
+const HELP_REGEX = /help|support|contact|faq/i;
+
+export async function runConsistency(ctx) {
+  const findings = [];
+  const max = Math.max(2, Math.min(ctx.maxPages || 5, 10));
+  const visited = new Set();
+  const fingerprints = [];
+  const queue = [ctx.url];
+  const startOrigin = new URL(ctx.url).origin;
+
+  while (queue.length && fingerprints.length < max) {
+    const next = queue.shift();
+    if (visited.has(next)) continue;
+    visited.add(next);
+
+    const sub = await ctx.context.newPage();
+    try {
+      await sub.goto(next, { waitUntil: "networkidle", timeout: 60_000 });
+      // Extra settle time for async client hydration (Clerk, Auth.js, etc.
+      // inject nav items after the initial networkidle event).
+      await sub.waitForTimeout(1500);
+      const fp = await sub.evaluate((helpRegexSource) => {
+        const helpRe = new RegExp(helpRegexSource, "i");
+        // Nav fingerprint: ordered list of link texts inside the first <nav>
+        const nav = document.querySelector("nav,[role='navigation']");
+        const navLinks = nav
+          ? Array.from(nav.querySelectorAll("a")).map((a) => (a.textContent || "").trim()).filter(Boolean).slice(0, 30)
+          : [];
+        // Components fingerprint: name (innerText) of every <button> + every link with role
+        const components = Array.from(document.querySelectorAll("button,a")).map((el) => {
+          const r = el.getBoundingClientRect();
+          return {
+            tag: el.tagName.toLowerCase(),
+            name: (el.innerText || el.getAttribute("aria-label") || "").trim().toLowerCase().slice(0, 50),
+            href: el.getAttribute("href") || "",
+            role: el.getAttribute("role") || "",
+            visible: r.width > 0 && r.height > 0,
+          };
+        }).filter((c) => c.visible && c.name).slice(0, 200);
+        // Help-link position: rect of the first link/button matching the regex
+        const helpEl = Array.from(document.querySelectorAll("a,button"))
+          .find((el) => helpRe.test(el.textContent || el.getAttribute("aria-label") || ""));
+        const helpRect = helpEl ? helpEl.getBoundingClientRect() : null;
+        return {
+          navLinks,
+          components,
+          helpRect: helpRect ? { x: Math.round(helpRect.x), y: Math.round(helpRect.y) } : null,
+        };
+      }, HELP_REGEX.source);
+
+      fingerprints.push({ url: next, ...fp });
+
+      // Enqueue same-origin internal links
+      const links = await sub.$$eval("a[href]", (els) => els.map((a) => a.href).filter(Boolean));
+      for (const l of links) {
+        try {
+          if (new URL(l).origin === startOrigin && !visited.has(l) && !l.includes("#")) queue.push(l);
+        } catch (_) {}
+        if (queue.length > 30) break;
+      }
+    } catch (_) {
+      // ignore page errors during crawl
+    } finally {
+      await sub.close().catch(() => {});
+    }
+  }
+
+  if (fingerprints.length < 2) {
+    return { findings }; // not enough pages crawled to compare
+  }
+
+  // ── 3.2.3 Consistent Navigation ──────────────────────────────────────
+  // Dedup each nav's links before comparing — many sites render the
+  // same nav twice (desktop + mobile dropdown) which produces duplicate
+  // entries. We only care about the RELATIVE order of unique link
+  // labels, not how many times each label appears.
+  function dedupPreserveOrder(arr) {
+    const seen = new Set();
+    const out = [];
+    for (const x of arr) {
+      if (!seen.has(x)) {
+        seen.add(x);
+        out.push(x);
+      }
+    }
+    return out;
+  }
+  const baseNav = dedupPreserveOrder(fingerprints[0].navLinks);
+  for (let i = 1; i < fingerprints.length; i++) {
+    const otherNav = dedupPreserveOrder(fingerprints[i].navLinks);
+    if (baseNav.length === 0 || otherNav.length === 0) continue;
+    // Find common nav items and check whether their relative order matches
+    const common = baseNav.filter((x) => otherNav.includes(x));
+    const baseOrder = common.map((x) => baseNav.indexOf(x));
+    const otherOrder = common.map((x) => otherNav.indexOf(x));
+    const ordered = baseOrder.every((v, j) => j === 0 || v > baseOrder[j - 1]);
+    const otherOrdered = otherOrder.every((v, j) => j === 0 || v > otherOrder[j - 1]);
+    if (!ordered || !otherOrdered) {
+      findings.push({
+        source: "playwright",
+        ruleId: "consistency/nav-order",
+        criteria: "3.2.3",
+        level: "AA",
+        impact: "moderate",
+        description: `Navigation links appear in a different order between ${fingerprints[0].url} and ${fingerprints[i].url}.`,
+        help: "Repeated navigation must occur in the same relative order on every page (WCAG 3.2.3).",
+        helpUrl: "https://www.w3.org/WAI/WCAG22/Understanding/consistent-navigation.html",
+        selector: "nav",
+        evidence: `Page 1 nav: ${JSON.stringify(baseNav)}\nPage ${i + 1} nav: ${JSON.stringify(otherNav)}`,
+        failureSummary: "Nav order differs",
+        screenshotFile: null,
+      });
+      break;
+    }
+  }
+
+  // ── 3.2.4 Consistent Identification ──────────────────────────────────
+  // For each href that appears on >=2 pages, check the names match.
+  //
+  // Soft-matching: WCAG 3.2.4 is about FUNCTIONAL consistency, not
+  // pixel-identical text. So we normalize (strip emoji, arrows,
+  // punctuation, collapse whitespace) and allow substring matches
+  // (e.g. "Enterprise" and "See enterprise plans" are compatible).
+  function normalize(s) {
+    return (s || "")
+      .toLowerCase()
+      .replace(/[→←▸▶️→↗↘🡒→]/g, "")
+      .replace(/[^\p{L}\p{N}\s]/gu, " ")
+      .replace(/\s+/g, " ")
+      .trim();
+  }
+  function compatible(a, b) {
+    const na = normalize(a);
+    const nb = normalize(b);
+    if (!na || !nb) return true;
+    if (na === nb) return true;
+    // Substring match (one extends the other — common CTA pattern)
+    if (na.includes(nb) || nb.includes(na)) return true;
+    // Same leading word (e.g. "Enterprise" and "Enterprise plans")
+    const firstA = na.split(" ")[0];
+    const firstB = nb.split(" ")[0];
+    if (firstA && firstA === firstB && firstA.length >= 4) return true;
+    return false;
+  }
+  const nameByHref = {};
+  for (const fp of fingerprints) {
+    for (const c of fp.components) {
+      if (!c.href) continue;
+      if (!nameByHref[c.href]) nameByHref[c.href] = new Set();
+      nameByHref[c.href].add(c.name);
+    }
+  }
+  const inconsistent = Object.entries(nameByHref)
+    .filter(([_, names]) => {
+      const arr = [...names];
+      if (arr.length < 2) return false;
+      // Flag only if at least one pair is NOT compatible
+      for (let i = 0; i < arr.length; i++) {
+        for (let j = i + 1; j < arr.length; j++) {
+          if (!compatible(arr[i], arr[j])) return true;
+        }
+      }
+      return false;
+    })
+    .slice(0, 10);
+  if (inconsistent.length) {
+    findings.push({
+      source: "playwright",
+      ruleId: "consistency/identification",
+      criteria: "3.2.4",
+      level: "AA",
+      impact: "moderate",
+      description: `${inconsistent.length} component(s) with the same destination have different visible names across pages.`,
+      help: "Components with the same function should be identified consistently. WCAG 3.2.4.",
+      helpUrl: "https://www.w3.org/WAI/WCAG22/Understanding/consistent-identification.html",
+      selector: "",
+      evidence: inconsistent.map(([href, names]) => `${href} → ${Array.from(names).join(" / ")}`).join("\n"),
+      failureSummary: `${inconsistent.length} components`,
+      screenshotFile: null,
+    });
+  }
+
+  // ── 3.2.6 Consistent Help ───────────────────────────────────────────
+  const helpRects = fingerprints.filter((f) => f.helpRect).map((f) => f.helpRect);
+  if (helpRects.length >= 2) {
+    const xs = helpRects.map((r) => r.x);
+    const ys = helpRects.map((r) => r.y);
+    const xRange = Math.max(...xs) - Math.min(...xs);
+    const yRange = Math.max(...ys) - Math.min(...ys);
+    if (xRange > 100 || yRange > 100) {
+      findings.push({
+        source: "playwright",
+        ruleId: "consistency/help-position",
+        criteria: "3.2.6",
+        level: "A",
+        impact: "moderate",
+        description: `Help mechanism appears in a different location across pages (x range ${xRange}px, y range ${yRange}px).`,
+        help: "WCAG 3.2.6 (added in 2.2) requires that any help mechanism (contact info, form, FAQ link, chatbot, self-help) appear in the same relative order on every page where it occurs.",
+        helpUrl: "https://www.w3.org/WAI/WCAG22/Understanding/consistent-help.html",
+        selector: "",
+        evidence: JSON.stringify(helpRects),
+        failureSummary: "Help link position varies",
+        screenshotFile: null,
+      });
+    }
+  }
+
+  return { findings };
+}

package/src/checkers/forms.js

@@ -0,0 +1,149 @@
+// Forms / error handling checker. Covers (substantially):
+//
+//   3.3.1 Error Identification — submit a form invalid, check for visible errors
+//   3.3.3 Error Suggestion     — check that errors offer a fix (AI-evaluated)
+//   3.3.4 Error Prevention     — check legal/financial forms have a confirm step
+//   3.3.7 Redundant Entry      — multi-step form duplicate field detection
+//
+// Strategy: enumerate forms, fill required fields with deliberately invalid
+// data (or leave them empty), submit, screenshot, capture aria-live /
+// inline error messages, and report.
+
+import { saveScreenshot, captureElement } from "../util/screenshot.js";
+import { askClaudeForJsonArray } from "../util/anthropic.js";
+
+export async function runForms(ctx) {
+  const { page } = ctx;
+  const findings = [];
+  const forms = await page.$$("form");
+  if (forms.length === 0) return { findings };
+
+  for (let fi = 0; fi < Math.min(forms.length, 5); fi++) {
+    const form = forms[fi];
+    let formInfo;
+    try {
+      formInfo = await form.evaluate((f) => {
+        const fields = Array.from(f.querySelectorAll("input,select,textarea"))
+          .filter((el) => !["submit", "button", "hidden"].includes(el.type))
+          .map((el) => ({
+            tag: el.tagName.toLowerCase(),
+            type: el.type || "",
+            name: el.name || "",
+            required: el.required,
+            id: el.id || "",
+          }));
+        const submit = f.querySelector("button[type=submit],input[type=submit],button:not([type])");
+        return {
+          action: f.action || "",
+          fields,
+          hasSubmit: !!submit,
+          html: f.outerHTML.slice(0, 1000),
+        };
+      });
+    } catch (_) { continue; }
+
+    if (!formInfo.hasSubmit || formInfo.fields.length === 0) continue;
+
+    // ── Pre-submission state ──
+    const beforeUrl = page.url();
+    const beforeText = await page.evaluate(() => document.body.innerText);
+
+    // Try a deliberately empty submission (worst case for required fields)
+    try {
+      await form.evaluate((f) => {
+        const btn = f.querySelector("button[type=submit],input[type=submit],button:not([type])");
+        btn?.click();
+      });
+      await page.waitForTimeout(800);
+    } catch (_) { continue; }
+
+    const afterText = await page.evaluate(() => document.body.innerText);
+    const navigated = page.url() !== beforeUrl;
+    const newText = afterText.slice(beforeText.length);
+    const errorPatterns = /(required|invalid|please|must|cannot be empty|enter|provide)/i;
+    const errorVisible = errorPatterns.test(newText) || (await page.evaluate(() => {
+      return Array.from(document.querySelectorAll("[role='alert'],[aria-live='assertive'],[aria-live='polite'],.error,.invalid,[aria-invalid='true']"))
+        .some((el) => (el.innerText || "").trim().length > 0);
+    }));
+
+    if (!navigated && !errorVisible) {
+      const buf = await captureElement(page, page.locator("form").nth(fi), { label: "Form: no error" });
+      const screenshotFile = await saveScreenshot(buf, ctx.screenshotsDir, "forms_no_error");
+      findings.push({
+        source: "playwright",
+        ruleId: "forms/no-error-on-invalid",
+        criteria: "3.3.1",
+        level: "A",
+        impact: "serious",
+        description: `Form #${fi + 1} accepted an empty submission silently — no visible error, no navigation, no aria-live announcement.`,
+        help: "When a user submits an invalid form, errors must be programmatically associated and visibly announced. Use aria-invalid + aria-describedby on the field, and a live region for the summary.",
+        helpUrl: "https://www.w3.org/WAI/WCAG22/Understanding/error-identification.html",
+        selector: `form:nth-of-type(${fi + 1})`,
+        evidence: formInfo.html.slice(0, 500),
+        failureSummary: "No error feedback after invalid submit",
+        screenshotFile,
+      });
+    } else if (errorVisible && ctx.ai.enabled) {
+      // ── 3.3.3 Error Suggestion — ask AI whether the error tells the user how to fix it
+      try {
+        const errSnapshot = await page.screenshot({ fullPage: false, type: "png" });
+        const prompt = `You are a WCAG 3.3.3 (Error Suggestion) auditor. The screenshot shows a form after an invalid submission. Evaluate whether the displayed error messages tell the user HOW TO FIX the problem (not just that something is wrong). Return a JSON array with ONE element: [ { "criterion": "3.3.3", "status": "pass" | "fail" | "needs-review", "summary": "...", "findings": [ { "issue": "...", "evidence": "quoted error text", "selector": "", "severity": "moderate" } ] } ]. Use 'pass' if every visible error provides a concrete suggestion. Use 'fail' if any error is generic ("invalid", "error", "required") with no fix guidance. Return ONLY the JSON.`;
+        const { array } = await askClaudeForJsonArray({
+          provider: ctx.ai.provider,
+          apiKey: ctx.ai.apiKey,
+          model: ctx.ai.model,
+          prompt,
+          images: [{ buffer: errSnapshot, mimeType: "image/png" }],
+        });
+        for (const r of array) {
+          if (r.status !== "fail") continue;
+          const screenshotFile = await saveScreenshot(errSnapshot, ctx.screenshotsDir, "forms_error_quality");
+          findings.push({
+            source: "ai",
+            ruleId: "ai/3.3.3-error-suggestion",
+            criteria: "3.3.3",
+            level: "AA",
+            impact: "moderate",
+            description: r.summary || "Form errors do not suggest how to fix the problem.",
+            help: (r.findings || []).map((f) => f.issue).join("\n"),
+            helpUrl: "https://www.w3.org/WAI/WCAG22/Understanding/error-suggestion.html",
+            selector: `form:nth-of-type(${fi + 1})`,
+            evidence: (r.findings || []).map((f) => f.evidence).join("\n"),
+            failureSummary: "Error message lacks fix guidance",
+            screenshotFile,
+          });
+        }
+      } catch (_) {}
+    }
+
+    // ── 3.3.7 Redundant Entry ─ rough check: count fields whose label/name
+    // suggests duplication (email vs confirm-email, password vs confirm-password)
+    // when no autofill help is offered.
+    const redundant = formInfo.fields.filter((f) =>
+      /confirm|repeat|verify|again/i.test(f.name + " " + f.id)
+    );
+    if (redundant.length) {
+      findings.push({
+        source: "playwright",
+        ruleId: "forms/redundant-entry",
+        criteria: "3.3.7",
+        level: "A",
+        impact: "moderate",
+        description: `Form requires the user to re-enter information (${redundant.map((r) => r.name || r.id).join(", ")}). WCAG 3.3.7 (added in 2.2) forbids requiring redundant entry within the same process unless essential.`,
+        help: "Replace 'confirm email/password' fields with a single field plus a 'show' toggle, or use the browser's autocomplete attributes.",
+        helpUrl: "https://www.w3.org/WAI/WCAG22/Understanding/redundant-entry.html",
+        selector: `form:nth-of-type(${fi + 1})`,
+        evidence: redundant.map((r) => `${r.tag}[name=${r.name}]`).join(", "),
+        failureSummary: `${redundant.length} confirm-style fields`,
+        screenshotFile: null,
+      });
+    }
+
+    // Navigate back if we left the page
+    if (page.url() !== beforeUrl) {
+      try { await page.goBack({ waitUntil: "networkidle" }); } catch (_) {}
+    }
+  }
+
+  return { findings };
+}

package/src/checkers/interaction.js

@@ -0,0 +1,142 @@
+// Interaction checker. Covers:
+//
+//   1.4.13 Content on Hover or Focus — hover each tooltip-trigger and verify
+//          dismissible (Esc), persistent (doesn't disappear when pointer
+//          moves to revealed content), hoverable
+//   3.2.1 On Focus — focus each control, watch for navigation/context change
+//   3.2.2 On Input — type into each input, watch for navigation/context change
+
+import { captureElement, saveScreenshot } from "../util/screenshot.js";
+
+export async function runInteraction(ctx) {
+  const { page } = ctx;
+  const findings = [];
+
+  // ── 1.4.13 Content on Hover/Focus ───────────────────────────────────
+  // Find candidate trigger elements: things with title attr, aria-describedby
+  // pointing to a hidden element, or [data-tooltip], common UI library hints.
+  const triggers = await page.evaluate(() => {
+    const sel = "[title]:not([title='']),[aria-describedby],[data-tooltip],[data-tippy-content],.tooltip,[role='tooltip']";
+    const arr = Array.from(document.querySelectorAll(sel)).slice(0, 30);
+    return arr.map((el, i) => {
+      const r = el.getBoundingClientRect();
+      return {
+        idx: i,
+        tag: el.tagName.toLowerCase(),
+        title: el.getAttribute("title") || el.getAttribute("data-tooltip") || "",
+        rect: { x: r.x + r.width / 2, y: r.y + r.height / 2 },
+      };
+    });
+  });
+
+  for (const t of triggers.slice(0, 15)) {
+    if (t.rect.x <= 0 || t.rect.y <= 0) continue;
+    try {
+      // Capture body innerText length before hover.
+      const before = await page.evaluate(() => document.body.innerText.length);
+      await page.mouse.move(t.rect.x, t.rect.y);
+      await page.waitForTimeout(400);
+      const afterHover = await page.evaluate(() => document.body.innerText.length);
+
+      if (afterHover - before < 5) continue; // nothing revealed; not a hover-content trigger
+
+      // Press Escape — content should disappear (dismissible)
+      await page.keyboard.press("Escape");
+      await page.waitForTimeout(200);
+      const afterEsc = await page.evaluate(() => document.body.innerText.length);
+      const dismissible = afterEsc < afterHover - 2;
+
+      if (!dismissible) {
+        findings.push({
+          source: "playwright",
+          ruleId: "interaction/hover-not-dismissible",
+          criteria: "1.4.13",
+          level: "AA",
+          impact: "moderate",
+          description: `Hover-revealed content was not dismissible via Escape on ${t.tag}.`,
+          help: "WCAG 1.4.13 requires that content shown on hover or focus be dismissable without moving the pointer (typically by Escape).",
+          helpUrl: "https://www.w3.org/WAI/WCAG22/Understanding/content-on-hover-or-focus.html",
+          selector: t.tag,
+          evidence: t.title,
+          failureSummary: "Esc did not remove the revealed content.",
+          screenshotFile: null,
+        });
+      }
+      // Reset
+      await page.mouse.move(0, 0);
+      await page.waitForTimeout(150);
+    } catch (_) {}
+  }
+
+  // ── 3.2.1 / 3.2.2 On Focus / On Input ──────────────────────────────
+  // Iterate the first ~15 form controls. For each: focus and watch URL.
+  // Then type a character and watch URL again.
+  const startUrl = page.url();
+  const controls = await page.$$eval("input,select,textarea,button", (els) =>
+    els.slice(0, 15).map((el) => {
+      function cssPath(node) {
+        const path = [];
+        while (node && node.nodeType === Node.ELEMENT_NODE && path.length < 6) {
+          let s = node.nodeName.toLowerCase();
+          if (node.id) { s += "#" + node.id; path.unshift(s); break; }
+          let sib = node, n = 1;
+          while ((sib = sib.previousElementSibling)) if (sib.nodeName === node.nodeName) n++;
+          if (n !== 1) s += `:nth-of-type(${n})`;
+          path.unshift(s);
+          node = node.parentElement;
+        }
+        return path.join(" > ");
+      }
+      return { tag: el.tagName.toLowerCase(), type: el.type || "", selector: cssPath(el) };
+    })
+  );
+
+  for (const c of controls) {
+    try {
+      const loc = page.locator(c.selector).first();
+      await loc.focus({ timeout: 1000 });
+      await page.waitForTimeout(150);
+      if (page.url() !== startUrl) {
+        findings.push({
+          source: "playwright",
+          ruleId: "interaction/on-focus-context-change",
+          criteria: "3.2.1",
+          level: "A",
+          impact: "serious",
+          description: `Focusing the ${c.tag} element triggered a navigation. WCAG 3.2.1 forbids context changes on focus.`,
+          help: "Move navigation behaviour to an explicit submit/click action, not focus.",
+          helpUrl: "https://www.w3.org/WAI/WCAG22/Understanding/on-focus.html",
+          selector: c.selector,
+          evidence: `Focus triggered navigation`,
+          failureSummary: `${startUrl} → ${page.url()}`,
+          screenshotFile: null,
+        });
+        return { findings };
+      }
+      if ((c.tag === "input" || c.tag === "textarea") && c.type !== "submit" && c.type !== "button") {
+        await loc.fill("a", { timeout: 1000 }).catch(() => {});
+        await page.waitForTimeout(150);
+        if (page.url() !== startUrl) {
+          findings.push({
+            source: "playwright",
+            ruleId: "interaction/on-input-context-change",
+            criteria: "3.2.2",
+            level: "A",
+            impact: "serious",
+            description: `Typing into the ${c.tag} element triggered a navigation. WCAG 3.2.2 forbids context changes on input without prior warning.`,
+            help: "Submit forms only on explicit user action, not on input event.",
+            helpUrl: "https://www.w3.org/WAI/WCAG22/Understanding/on-input.html",
+            selector: c.selector,
+            evidence: `Input triggered navigation`,
+            failureSummary: `${startUrl} → ${page.url()}`,
+            screenshotFile: null,
+          });
+          return { findings };
+        }
+        await loc.fill("").catch(() => {});
+      }
+    } catch (_) {}
+  }
+
+  return { findings };
+}