@wcag-checkr/ci 1.0.0-rc.13

package/wcagcheckr-ci.mjs

@@ -0,0 +1,559 @@
+#!/usr/bin/env node
+// wcagcheckr CI runner — drives the wcagcheckr Chrome extension via Playwright
+// to run full-page accessibility audits in a headless-ish CI environment.
+//
+// Why this exists: competing tools (axe DevTools Pro, Siteimprove) ship CI
+// runners as paid features. We ship one for free. Reuses the same audit
+// engine + state matrix the extension uses interactively, so the CI run
+// matches what a developer sees in the side panel.
+//
+// Usage:
+//   wcagcheckr-ci audit <url> [options]
+//
+// Options:
+//   --format <json|sarif|junit>     Output format (default: json)
+//   --output <file>                 Write output to file (default: stdout)
+//   --threshold <none|critical|serious|moderate|minor>
+//                                    Exit non-zero if any violations at or above
+//                                    this severity (default: serious)
+//   --extension-dir <path>          Path to a built dist/ (default: bundled)
+//   --timeout <ms>                  Audit timeout (default: 120000)
+//   --quiet                         Suppress progress output to stderr
+//
+// Exit codes:
+//   0  — success, threshold not exceeded
+//   1  — threshold exceeded (CI failure signal)
+//   2  — runtime error (extension didn't load, target unreachable, etc.)
+
+import { chromium } from 'playwright';
+import { existsSync, readFileSync, writeFileSync } from 'node:fs';
+import { resolve, dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { createHash, createPublicKey, verify as cryptoVerify } from 'node:crypto';
+
+const ROOT = resolve(dirname(fileURLToPath(import.meta.url)));
+// Published package layout: cli/dist/ is bundled inside the package root
+// (see scripts/prepare-cli-publish.mjs). Dev layout: dist/ is at the repo
+// root, one directory up from this script. Prefer the bundled copy so
+// `npx @wcag-checkr/ci` works without a separate dist install.
+const BUNDLED_DIST = resolve(ROOT, 'dist');
+const DEV_DIST = resolve(ROOT, '..', 'dist');
+const DEFAULT_EXT_DIR = existsSync(BUNDLED_DIST) ? BUNDLED_DIST : DEV_DIST;
+
+const SEVERITY_RANK = { critical: 0, serious: 1, moderate: 2, minor: 3 };
+
+function parseArgs(argv) {
+  const args = {
+    command: null,
+    url: null,
+    inputFile: null,
+    format: 'json',
+    threshold: 'serious',
+    timeout: 120_000,
+    quiet: false,
+    license: null,
+    publicKeyUrl: 'https://api.wcagcheckr.com/v1/products/wcagcheckr/forensic/public-key',
+  };
+  for (let i = 2; i < argv.length; i++) {
+    const a = argv[i];
+    if (!args.command && a !== '--help' && !a.startsWith('--')) {
+      args.command = a;
+      continue;
+    }
+    if (args.command === 'audit' && !args.url && !a.startsWith('--')) {
+      args.url = a;
+      continue;
+    }
+    if (args.command === 'verify' && !args.inputFile && !a.startsWith('--')) {
+      args.inputFile = a;
+      continue;
+    }
+    if (a === '--format') args.format = argv[++i];
+    else if (a === '--output') args.output = argv[++i];
+    else if (a === '--threshold') args.threshold = argv[++i];
+    else if (a === '--extension-dir') args.extensionDir = argv[++i];
+    else if (a === '--timeout') args.timeout = parseInt(argv[++i], 10);
+    else if (a === '--quiet') args.quiet = true;
+    else if (a === '--license') args.license = argv[++i];
+    else if (a === '--public-key-url') args.publicKeyUrl = argv[++i];
+    else if (a === '--help' || a === '-h') args.help = true;
+  }
+  return args;
+}
+
+function printUsage() {
+  console.log(`wcagcheckr-ci — headless WCAG audit runner
+
+Usage:
+  wcagcheckr-ci audit <url> [options]
+  wcagcheckr-ci verify <forensic-log.json> [options]
+
+audit options:
+  --format <json|sarif|junit>     Output format (default: json)
+  --output <file>                 Write output to file (default: stdout)
+  --threshold <none|critical|serious|moderate|minor>
+                                  Severity below which exit code stays 0 (default: serious)
+  --extension-dir <path>          Path to a built dist/ (default: bundled)
+  --timeout <ms>                  Audit timeout (default: 120000)
+  --license <token>               Activate this license token before auditing
+                                  (gates paid features like forensic anchoring)
+  --quiet                         Suppress progress output
+
+verify options:
+  --public-key-url <url>          Override the public-key endpoint (default:
+                                  https://api.wcagcheckr.com/.../forensic/public-key)
+  --quiet                         Only print per-entry FAIL lines + final summary
+
+Exit codes:
+  0  ok    1  threshold exceeded / verification failed    2  runtime error`);
+}
+
+const args = parseArgs(process.argv);
+
+if (args.help || !args.command) {
+  printUsage();
+  process.exit(args.help ? 0 : 2);
+}
+
+const log = (...m) => {
+  if (!args.quiet) console.error('→', ...m);
+};
+
+// ─── verify subcommand ──────────────────────────────────────────────────────
+//
+// Reads a forensic-log JSON export from the extension's Forensic tab and
+// validates each entry's identity hash + ed25519 server signature offline.
+// RFC 3161 timestamp tokens are surfaced (with a verification command hint)
+// rather than parsed here — full ASN.1 parsing is out-of-scope for the v0
+// verifier; users run `openssl ts -verify` against FreeTSA's chain.
+
+if (args.command === 'verify') {
+  if (!args.inputFile) {
+    console.error('✗ verify: missing input file path.\n');
+    printUsage();
+    process.exit(2);
+  }
+  await runVerify(args);
+  // runVerify exits the process itself.
+}
+
+if (args.command !== 'audit' || !args.url) {
+  console.error(`✗ Unknown command or missing URL.\n`);
+  printUsage();
+  process.exit(2);
+}
+
+const EXT_DIR = args.extensionDir ?? DEFAULT_EXT_DIR;
+if (!existsSync(EXT_DIR)) {
+  console.error(`✗ Extension directory not found: ${EXT_DIR}`);
+  console.error(`  Either run from a checkout with dist/ built, or pass --extension-dir.`);
+  process.exit(2);
+}
+
+if (!['json', 'sarif', 'junit'].includes(args.format)) {
+  console.error(`✗ Unknown format: ${args.format}. Must be json, sarif, or junit.`);
+  process.exit(2);
+}
+
+if (!['none', 'critical', 'serious', 'moderate', 'minor'].includes(args.threshold)) {
+  console.error(`✗ Unknown threshold: ${args.threshold}.`);
+  process.exit(2);
+}
+
+log(`Loading extension from ${EXT_DIR}`);
+log(`Auditing ${args.url} (threshold: ${args.threshold}, format: ${args.format})`);
+
+// Chrome extensions cannot load in legacy `headless: true` mode, so we
+// pass `--headless=new` as a Chromium arg (the modern headless mode that
+// DOES support extensions). Playwright's `headless` option needs a real
+// boolean — earlier code passed the string 'new' which crashed launch
+// outright with "headless: expected boolean, got string".
+const context = await chromium.launchPersistentContext('', {
+  headless: false,
+  viewport: { width: 1280, height: 800 },
+  args: [
+    `--disable-extensions-except=${EXT_DIR}`,
+    `--load-extension=${EXT_DIR}`,
+    '--headless=new',
+  ],
+});
+
+let exitCode = 0;
+try {
+  const [targetPage] = context.pages();
+  await targetPage.goto(args.url, { waitUntil: 'domcontentloaded', timeout: 30_000 });
+  log('target page loaded');
+
+  let extId;
+  for (let i = 0; i < 40 && !extId; i++) {
+    const m = context.serviceWorkers()[0]?.url().match(/^chrome-extension:\/\/([^/]+)\//);
+    if (m) extId = m[1];
+    if (!extId) await new Promise((r) => setTimeout(r, 250));
+  }
+  if (!extId) throw new Error('extension SW did not register within 10s');
+  log(`extension loaded (id: ${extId.slice(0, 8)}…)`);
+
+  const sidePanel = await context.newPage();
+  await sidePanel.goto(`chrome-extension://${extId}/side-panel/side-panel.html`);
+
+  // Headless persona — pick dev mode if wizard appears.
+  const devButton = sidePanel.getByRole('button', { name: /developer or agency/i });
+  if (await devButton.isVisible({ timeout: 5_000 }).catch(() => false)) {
+    await devButton.click();
+    await sidePanel.waitForTimeout(500);
+  }
+  await sidePanel.waitForSelector('h1:has-text("wcagcheckr")', { timeout: 10_000 });
+
+  // Activate license token, if provided. Gated paid features (forensic
+  // anchoring, AI-summary, etc.) need an active license at audit time.
+  if (args.license) {
+    log('activating license token');
+    const activated = await sidePanel.evaluate(
+      async (token) =>
+        await new Promise((resolve) => {
+          chrome.runtime.sendMessage(
+            { type: 'LICENSE_SET_REQUEST', token },
+            (response) => resolve(response ?? null),
+          );
+        }),
+      args.license,
+    );
+    if (!activated || activated.tier === 'free') {
+      throw new Error(
+        `license activation did not yield a paid tier — got ${JSON.stringify(activated)}`,
+      );
+    }
+    log(`license active — tier: ${activated.tier}`);
+  }
+
+  // Skip onboarding tour
+  for (let i = 0; i < 5; i++) {
+    const gotIt = sidePanel.getByRole('button', { name: 'Got it' });
+    const next = sidePanel.getByRole('button', { name: 'Next' });
+    if (await gotIt.isVisible({ timeout: 500 }).catch(() => false)) {
+      await gotIt.click();
+      break;
+    }
+    if (await next.isVisible({ timeout: 500 }).catch(() => false)) {
+      await next.click();
+      await sidePanel.waitForTimeout(150);
+      continue;
+    }
+    break;
+  }
+
+  await sidePanel.getByLabel('Audit mode').selectOption('full-page');
+  await targetPage.bringToFront();
+  await targetPage.waitForTimeout(500);
+  await sidePanel.bringToFront();
+  await sidePanel.waitForTimeout(300);
+  await sidePanel.getByRole('button', { name: /Scan page|Auditing/ }).click();
+  log('audit started');
+
+  await sidePanel
+    .getByText(/\d+\s+states?\s+·\s+\d+\s+unique\s+violations?/, { exact: false })
+    .waitFor({ timeout: args.timeout });
+  log('audit completed');
+
+  // Trigger the existing export pipeline. The SW handles EXPORT_REQUEST and
+  // returns the formatted content for the requested format.
+  const formatMap = { json: 'json', sarif: 'sarif', junit: 'junit' };
+  const exportFormat = formatMap[args.format];
+
+  const result = await sidePanel.evaluate(async (fmt) => {
+    // Read the just-completed audit from chrome.storage.local. The side-
+    // panel persists results there in wire-messaging.ts when AUDIT_COMPLETE_
+    // EVENT fires, so by the time the matrix-summary text is visible (which
+    // we waited on above), the data is in storage. Earlier code tried to
+    // read from `window.__wcagcheckrStore` — that global is never set, so
+    // every CLI audit silently returned 0 violations.
+    const stored = await new Promise((resolve) => {
+      chrome.storage.local.get('sidePanel:lastAudit', (r) =>
+        resolve(r['sidePanel:lastAudit'] ?? null),
+      );
+    });
+    const results = stored?.results ?? [];
+    const delta = stored?.delta ?? null;
+    return await new Promise((resolve, reject) => {
+      const timeoutId = setTimeout(() => reject(new Error('export timeout')), 30_000);
+      chrome.runtime.sendMessage(
+        { type: 'EXPORT_REQUEST', format: fmt, results, delta: delta ?? undefined },
+        (response) => {
+          clearTimeout(timeoutId);
+          if (chrome.runtime.lastError) {
+            reject(new Error(chrome.runtime.lastError.message));
+            return;
+          }
+          resolve(response);
+        },
+      );
+    });
+  }, exportFormat);
+
+  if (!result || !result.content) {
+    throw new Error('export returned no content');
+  }
+
+  // Severity counts for threshold check — same chrome.storage.local source
+  // as the export above (window.__wcagcheckrStore is never defined).
+  const counts = await sidePanel.evaluate(async () => {
+    const stored = await new Promise((resolve) => {
+      chrome.storage.local.get('sidePanel:lastAudit', (r) =>
+        resolve(r['sidePanel:lastAudit'] ?? null),
+      );
+    });
+    const results = stored?.results ?? [];
+    const totals = { critical: 0, serious: 0, moderate: 0, minor: 0 };
+    const seen = new Set();
+    for (const r of results) {
+      for (const v of r.violations ?? []) {
+        const key = (v.matchKey ?? '') + ':' + (v.target?.selector ?? '');
+        if (seen.has(key)) continue;
+        seen.add(key);
+        if (v.impact in totals) totals[v.impact]++;
+      }
+    }
+    return totals;
+  });
+
+  log(`unique violations: critical=${counts.critical}, serious=${counts.serious}, moderate=${counts.moderate}, minor=${counts.minor}`);
+
+  if (args.output) {
+    writeFileSync(args.output, result.content, 'utf8');
+    log(`output written to ${args.output} (${result.content.length} bytes)`);
+  } else {
+    process.stdout.write(result.content);
+    if (!result.content.endsWith('\n')) process.stdout.write('\n');
+  }
+
+  if (args.threshold !== 'none') {
+    const cap = SEVERITY_RANK[args.threshold];
+    const exceeds =
+      (cap >= SEVERITY_RANK.critical && counts.critical > 0) ||
+      (cap >= SEVERITY_RANK.serious && counts.serious > 0) ||
+      (cap >= SEVERITY_RANK.moderate && counts.moderate > 0) ||
+      (cap >= SEVERITY_RANK.minor && counts.minor > 0);
+    if (exceeds) {
+      log(`✗ threshold exceeded (--threshold=${args.threshold})`);
+      exitCode = 1;
+    } else {
+      log(`✓ threshold not exceeded (--threshold=${args.threshold})`);
+    }
+  }
+} catch (err) {
+  console.error(`✗ ${err.message}`);
+  if (err.stack && !args.quiet) console.error(err.stack);
+  exitCode = 2;
+} finally {
+  await context.close();
+}
+
+process.exit(exitCode);
+
+// ─── verify subcommand impl ────────────────────────────────────────────────
+
+/** Deterministic canonical JSON — keys sorted alphabetically, no whitespace.
+ *  Same algorithm as forensic-log.ts in the extension. Critical: the hash
+ *  and signature payloads must serialize byte-for-byte identically across
+ *  extension, server, and CLI, or the verification fails. */
+function canonicalJson(value) {
+  if (value === null || typeof value !== 'object') return JSON.stringify(value);
+  if (Array.isArray(value)) return '[' + value.map(canonicalJson).join(',') + ']';
+  const keys = Object.keys(value).sort();
+  return (
+    '{' +
+    keys.map((k) => JSON.stringify(k) + ':' + canonicalJson(value[k])).join(',') +
+    '}'
+  );
+}
+
+function sha256Hex(text) {
+  return createHash('sha256').update(text, 'utf8').digest('hex');
+}
+
+async function fetchPublicKey(url, quiet) {
+  if (!quiet) console.error('→', 'fetching server public key from', url);
+  const res = await fetch(url);
+  if (!res.ok) throw new Error(`public-key fetch HTTP ${res.status}`);
+  const json = await res.json();
+  if (!json.publicKeyBase64 || !json.fingerprint) {
+    throw new Error('public-key response missing publicKeyBase64 / fingerprint');
+  }
+  return { publicKeyBase64: json.publicKeyBase64, fingerprint: json.fingerprint };
+}
+
+function verifyEd25519(messageBytes, signatureBytes, spkiBuffer) {
+  // Node's createPublicKey accepts an SPKI DER buffer when format='der',
+  // type='spki'. ed25519 verify() takes (algorithm=null, data, key, signature).
+  const key = createPublicKey({ key: spkiBuffer, format: 'der', type: 'spki' });
+  return cryptoVerify(null, messageBytes, key, signatureBytes);
+}
+
+async function runVerify(args) {
+  const path = resolve(args.inputFile);
+  if (!existsSync(path)) {
+    console.error(`✗ verify: file not found: ${path}`);
+    process.exit(2);
+  }
+  let payload;
+  try {
+    payload = JSON.parse(readFileSync(path, 'utf8'));
+  } catch (err) {
+    console.error(`✗ verify: failed to parse JSON: ${err.message}`);
+    process.exit(2);
+  }
+  if (!Array.isArray(payload.entries)) {
+    console.error('✗ verify: input is not a wcagcheckr forensic-log export (entries[] missing).');
+    process.exit(2);
+  }
+
+  const entries = payload.entries;
+  if (!args.quiet) {
+    console.error('→', `loaded ${entries.length} entr${entries.length === 1 ? 'y' : 'ies'} from ${path}`);
+    console.error('→', `tool: ${payload.tool?.name ?? '?'} ${payload.tool?.version ?? ''}`);
+    console.error('→', `exported: ${payload.exportedAt ?? 'n/a'}`);
+  }
+
+  // Lazy-fetch the public key only if at least one entry has a receipt.
+  let cachedKey = null;
+  async function getKey() {
+    if (cachedKey) return cachedKey;
+    cachedKey = await fetchPublicKey(args.publicKeyUrl, args.quiet);
+    cachedKey.spkiBuffer = Buffer.from(cachedKey.publicKeyBase64, 'base64');
+    return cachedKey;
+  }
+
+  let hashOk = 0;
+  let hashFail = 0;
+  let sigOk = 0;
+  let sigFail = 0;
+  let sigSkipped = 0;
+  let chainOk = 0;
+  let chainFail = 0;
+  let chainUnverifiable = 0;
+  let chainSkipped = 0;
+
+  // The server's anchor chain is per-product, not per-component. We can only
+  // hard-verify a link when the claimed predecessor hash exists in THIS
+  // export. Otherwise the chain is unverifiable from the file alone (the
+  // user would need to export the full history to verify deep chain links).
+  const hashesInExport = new Set(entries.map((e) => e.hash));
+
+  // Sort chronologically for stable per-entry output.
+  const sorted = [...entries].sort((a, b) => a.capturedAt.localeCompare(b.capturedAt));
+
+  for (const e of sorted) {
+    const tag = `${e.componentId}@${e.capturedAt}`;
+
+    // 1. Recompute the identity hash.
+    const hashInput = {
+      componentId: e.componentId,
+      pageUrl: e.pageUrl,
+      scope: e.scope,
+      grade: e.grade,
+      totals: e.totals,
+      axeVersion: e.axeVersion,
+      capturedAt: e.capturedAt,
+      statesAudited: e.statesAudited,
+    };
+    const recomputed = sha256Hex(canonicalJson(hashInput));
+    if (recomputed === e.hash) {
+      hashOk++;
+      if (!args.quiet) console.log(`  HASH OK    ${tag}  ${e.hash.slice(0, 12)}…`);
+    } else {
+      hashFail++;
+      console.log(`  HASH FAIL  ${tag}`);
+      console.log(`             stored=${e.hash}`);
+      console.log(`             recomp=${recomputed}`);
+    }
+
+    // 2. Verify the server signature (if a receipt exists).
+    if (e.receipt) {
+      try {
+        const key = await getKey();
+        if (key.fingerprint !== e.receipt.serverKeyFingerprint) {
+          // Different key — caller may want to pass --public-key-url for a
+          // historical key. We don't fail outright; surface it.
+          console.log(
+            `  SIG FAIL   ${tag}  key fingerprint mismatch (entry=${e.receipt.serverKeyFingerprint.slice(0, 12)}…, server=${key.fingerprint.slice(0, 12)}…)`,
+          );
+          sigFail++;
+        } else {
+          const payloadObj = {
+            hash: e.hash,
+            anchoredAt: e.receipt.anchoredAt,
+            tsaName: e.receipt.tsaName,
+            productSlug: payload.tool?.name ?? 'wcagcheckr',
+            prevAuditHash: e.receipt.prevAuditHash ?? null,
+          };
+          if (e.receipt.schemaVersion === 1 || !e.receipt.prevAuditHash) {
+            delete payloadObj.prevAuditHash;
+          }
+          const canonical = canonicalJson(payloadObj);
+          const bytes = Buffer.from(canonical, 'utf8');
+          const sig = Buffer.from(e.receipt.serverSignatureBase64, 'base64');
+          const ok = verifyEd25519(bytes, sig, key.spkiBuffer);
+          if (ok) {
+            sigOk++;
+            if (!args.quiet) console.log(`  SIG OK     ${tag}  ed25519 (${e.receipt.tsaName})`);
+          } else {
+            sigFail++;
+            console.log(`  SIG FAIL   ${tag}  ed25519 verification failed`);
+          }
+        }
+      } catch (err) {
+        sigFail++;
+        console.log(`  SIG ERR    ${tag}  ${err.message}`);
+      }
+
+      // 3. Chain check — v2 receipts include prevAuditHash linking to the
+      //    previous anchor for the product. We can hard-verify a link only
+      //    when the predecessor hash is present in THIS export AND its
+      //    signature already verified above. Otherwise the chain link is
+      //    unverifiable from the file alone (deeper history not exported).
+      if (e.receipt.schemaVersion === 2 && typeof e.receipt.prevAuditHash === 'string') {
+        const prev = e.receipt.prevAuditHash;
+        const genesisHash = '0'.repeat(64);
+        if (prev === genesisHash) {
+          // Sentinel — this anchor claims to be the first ever for the product.
+          chainOk++;
+          if (!args.quiet) console.log(`  CHAIN OK   ${tag}  product-genesis anchor`);
+        } else if (hashesInExport.has(prev)) {
+          // The predecessor is in this export — and since we verified above
+          // that its hash matched its identifying fields, the chain link is
+          // sound: the receipt's signed bytes commit to that specific hash.
+          chainOk++;
+          if (!args.quiet) console.log(`  CHAIN OK   ${tag}  predecessor ${prev.slice(0, 12)}… present`);
+        } else {
+          chainUnverifiable++;
+          if (!args.quiet) {
+            console.log(
+              `  CHAIN ??   ${tag}  predecessor ${prev.slice(0, 12)}… not in this export — re-export full history to hard-verify`,
+            );
+          }
+        }
+      } else {
+        chainSkipped++;
+      }
+    } else {
+      sigSkipped++;
+      if (!args.quiet) console.log(`  SIG SKIP   ${tag}  local-only entry (no receipt)`);
+    }
+  }
+
+  // Summary.
+  console.error('');
+  console.error(`Hash:      ${hashOk} ok · ${hashFail} fail`);
+  console.error(`Signature: ${sigOk} ok · ${sigFail} fail · ${sigSkipped} skipped (local-only)`);
+  console.error(`Chain:     ${chainOk} ok · ${chainFail} fail · ${chainUnverifiable} unverifiable (predecessor not in export) · ${chainSkipped} skipped (v1 / local-only)`);
+  if (sigOk > 0) {
+    console.error('');
+    console.error('Note: RFC 3161 TimeStampToken bytes are present in each anchored entry but');
+    console.error('      this verifier only checks the ed25519 server signature. To fully verify');
+    console.error('      the TSA timestamp, save the rfc3161TokenBase64 to a .tsr file and run:');
+    console.error('      openssl ts -verify -data <audit-bytes> -in token.tsr -CAfile freetsa.pem');
+  }
+
+  const failed = hashFail + sigFail + chainFail;
+  process.exit(failed === 0 ? 0 : 1);
+}