@monoes/monobrowse 1.0.11 → 1.0.13

package/src/report/analyze.ts

@@ -0,0 +1,181 @@
+/**
+ * Budget evaluation — the step that makes "the agent tested it" mean
+ * something. Pure function of CaptureData + Budget, so CI behaviour is
+ * testable without a browser.
+ */
+
+import { countA11y } from './a11y.js';
+import { budgetLabel, formatThreshold } from './budget.js';
+import type {
+  A11yFinding,
+  Budget,
+  BudgetFailure,
+  CaptureData,
+  ConsoleEntry,
+  Report,
+  RequestEntry,
+  VerdictValue,
+} from './types.js';
+
+/** How many concrete examples to quote in a failure's `detail`. */
+const DETAIL_SAMPLE = 3;
+
+export function consoleErrors(entries: ConsoleEntry[]): ConsoleEntry[] {
+  return entries.filter((e) => e.type === 'error');
+}
+
+export function consoleWarnings(entries: ConsoleEntry[]): ConsoleEntry[] {
+  return entries.filter((e) => e.type === 'warn' || e.type === 'warning');
+}
+
+export function failedRequests(requests: RequestEntry[]): RequestEntry[] {
+  return requests.filter((r) => r.failed);
+}
+
+function truncate(text: string, max = 160): string {
+  const flat = text.replace(/\s+/g, ' ').trim();
+  return flat.length > max ? `${flat.slice(0, max - 1)}…` : flat;
+}
+
+function sample(items: string[]): string {
+  // Not `.map(truncate)`: map passes the index as the second argument, which
+  // would become truncate's `max` and shave every sample down to nothing.
+  const shown = items.slice(0, DETAIL_SAMPLE).map((item) => truncate(item));
+  const rest = items.length - shown.length;
+  return rest > 0 ? `${shown.join(' | ')} (+${rest} more)` : shown.join(' | ');
+}
+
+function describeRequest(r: RequestEntry): string {
+  const status = r.status !== undefined ? String(r.status) : (r.errorText ?? 'no response');
+  return `${status} ${r.method} ${r.url}`;
+}
+
+interface CountCheck {
+  key: keyof Budget;
+  actual: number;
+  detail: string;
+}
+
+interface MetricCheck {
+  key: keyof Budget;
+  actual: number | undefined;
+  format: (v: number) => string;
+}
+
+export interface Evaluation {
+  verdict: VerdictValue;
+  failures: BudgetFailure[];
+  /** Budgets that were set but could not be checked — the page never reported the metric. */
+  unmeasured: string[];
+}
+
+export function evaluateBudget(data: CaptureData, budget: Budget): Evaluation {
+  const failures: BudgetFailure[] = [];
+  const unmeasured: string[] = [];
+
+  const cErrors = consoleErrors(data.console);
+  const failed = failedRequests(data.requests);
+  const a11y = countA11y(data.a11y);
+
+  const counts: CountCheck[] = [
+    {
+      key: 'maxConsoleErrors',
+      actual: cErrors.length,
+      detail: sample(cErrors.map((e) => e.text)),
+    },
+    {
+      key: 'maxPageErrors',
+      actual: data.pageErrors.length,
+      detail: sample(data.pageErrors.map((e) => e.text)),
+    },
+    {
+      key: 'maxFailedRequests',
+      actual: failed.length,
+      detail: sample(failed.map(describeRequest)),
+    },
+    {
+      key: 'maxA11yErrors',
+      actual: a11y.errors,
+      detail: sample(
+        data.a11y
+          .filter((f) => f.impact === 'error')
+          .map((f: A11yFinding) => `${f.rule} at ${f.locator}`),
+      ),
+    },
+  ];
+
+  for (const check of counts) {
+    const limit = budget[check.key];
+    if (limit === null) continue;
+    if (check.actual > limit) {
+      failures.push({
+        budget: check.key,
+        expected: formatThreshold(check.key, limit),
+        actual: String(check.actual),
+        detail: check.detail || undefined,
+      });
+    }
+  }
+
+  const ms = (v: number) => `${Math.round(v)}ms`;
+  const score = (v: number) => v.toFixed(4).replace(/0+$/, '').replace(/\.$/, '');
+  const metrics: MetricCheck[] = [
+    { key: 'lcpMs', actual: data.vitals.lcp, format: ms },
+    { key: 'clsScore', actual: data.vitals.cls, format: score },
+    { key: 'inpMs', actual: data.vitals.inp, format: ms },
+    { key: 'fcpMs', actual: data.vitals.fcp, format: ms },
+    { key: 'ttfbMs', actual: data.vitals.ttfb, format: ms },
+  ];
+
+  for (const metric of metrics) {
+    const limit = budget[metric.key];
+    if (limit === null) continue;
+    if (metric.actual === undefined || !Number.isFinite(metric.actual)) {
+      // A page that never fires an LCP entry (no contentful paint, an
+      // observer the browser does not support, a collector that timed out)
+      // must not be silently graded as passing — but it is not a breach
+      // either. Surface it as unmeasured and let the report say so.
+      unmeasured.push(budgetLabel(metric.key));
+      continue;
+    }
+    if (metric.actual > limit) {
+      failures.push({
+        budget: metric.key,
+        expected: formatThreshold(metric.key, limit),
+        actual: metric.format(metric.actual),
+      });
+    }
+  }
+
+  return { verdict: failures.length > 0 ? 'fail' : 'pass', failures, unmeasured };
+}
+
+export function buildReport(data: CaptureData, budget: Budget): Report {
+  const evaluation = evaluateBudget(data, budget);
+  const a11y = countA11y(data.a11y);
+  return {
+    ...data,
+    budget,
+    verdict: evaluation.verdict,
+    failures: evaluation.failures,
+    unmeasured: evaluation.unmeasured,
+    counts: {
+      consoleErrors: consoleErrors(data.console).length,
+      consoleWarnings: consoleWarnings(data.console).length,
+      pageErrors: data.pageErrors.length,
+      requests: data.requests.length,
+      failedRequests: failedRequests(data.requests).length,
+      a11yErrors: a11y.errors,
+      a11yWarnings: a11y.warnings,
+    },
+  };
+}
+
+/** One line for a CI log / agent transcript. */
+export function summarizeVerdict(report: Report): string {
+  if (report.verdict === 'pass') {
+    return `PASS — ${report.counts.requests} requests, 0 budget failures`;
+  }
+  const names = report.failures.map((f) => budgetLabel(f.budget)).join(', ');
+  return `FAIL — ${report.failures.length} budget failure(s): ${names}`;
+}

package/src/report/budget.ts

@@ -0,0 +1,155 @@
+/**
+ * Budget parsing for `monobrowse report`.
+ *
+ * A budget is what turns "the agent looked at the page" into "the agent
+ * tested the page": the thresholds the run is gated on. Defaults are the
+ * strict ones you actually want in CI — zero errors, zero failed requests,
+ * Core Web Vitals at Google's "good" boundary.
+ */
+
+import { readFile } from 'node:fs/promises';
+import type { Budget } from './types.js';
+
+export const DEFAULT_BUDGET: Budget = {
+  maxConsoleErrors: 0,
+  maxPageErrors: 0,
+  maxFailedRequests: 0,
+  maxA11yErrors: 0,
+  lcpMs: 2500,
+  clsScore: 0.1,
+  inpMs: 200,
+  fcpMs: null,
+  ttfbMs: null,
+};
+
+/** Every spelling we accept, normalized (lowercased, `_`/`-` stripped). */
+const KEY_ALIASES: Record<string, keyof Budget> = {
+  maxconsoleerrors: 'maxConsoleErrors',
+  consoleerrors: 'maxConsoleErrors',
+  maxpageerrors: 'maxPageErrors',
+  pageerrors: 'maxPageErrors',
+  maxfailedrequests: 'maxFailedRequests',
+  failedrequests: 'maxFailedRequests',
+  maxa11yerrors: 'maxA11yErrors',
+  a11yerrors: 'maxA11yErrors',
+  maxaccessibilityerrors: 'maxA11yErrors',
+  lcp: 'lcpMs',
+  lcpms: 'lcpMs',
+  cls: 'clsScore',
+  clsscore: 'clsScore',
+  inp: 'inpMs',
+  inpms: 'inpMs',
+  fcp: 'fcpMs',
+  fcpms: 'fcpMs',
+  ttfb: 'ttfbMs',
+  ttfbms: 'ttfbMs',
+};
+
+const BUDGET_UNITS: Record<keyof Budget, 'ms' | 'score' | 'count'> = {
+  maxConsoleErrors: 'count',
+  maxPageErrors: 'count',
+  maxFailedRequests: 'count',
+  maxA11yErrors: 'count',
+  lcpMs: 'ms',
+  clsScore: 'score',
+  inpMs: 'ms',
+  fcpMs: 'ms',
+  ttfbMs: 'ms',
+};
+
+const BUDGET_LABELS: Record<keyof Budget, string> = {
+  maxConsoleErrors: 'Console errors',
+  maxPageErrors: 'Uncaught page errors',
+  maxFailedRequests: 'Failed requests',
+  maxA11yErrors: 'Accessibility errors',
+  lcpMs: 'LCP (Largest Contentful Paint)',
+  clsScore: 'CLS (Cumulative Layout Shift)',
+  inpMs: 'INP (Interaction to Next Paint)',
+  fcpMs: 'FCP (First Contentful Paint)',
+  ttfbMs: 'TTFB (Time to First Byte)',
+};
+
+export function budgetLabel(key: keyof Budget): string {
+  return BUDGET_LABELS[key];
+}
+
+/** `2500` -> `<= 2500ms`, `0` -> `<= 0`, `0.1` -> `<= 0.1`. */
+export function formatThreshold(key: keyof Budget, value: number): string {
+  const unit = BUDGET_UNITS[key];
+  if (unit === 'ms') return `<= ${value}ms`;
+  return `<= ${value}`;
+}
+
+function normalizeKey(raw: string): string {
+  return raw.toLowerCase().replace(/[_\-\s]/g, '');
+}
+
+/**
+ * Turn a parsed JSON object into a Budget, merged over the defaults.
+ *
+ * Unknown keys throw rather than being ignored: a typo'd `max_console_error`
+ * silently loosening a CI gate is exactly the failure mode budgets exist to
+ * prevent.
+ */
+export function parseBudget(input: unknown, source = 'budget'): Budget {
+  if (input === null || typeof input !== 'object' || Array.isArray(input)) {
+    throw new Error(`Invalid ${source}: expected a JSON object of budget keys`);
+  }
+
+  const budget: Budget = { ...DEFAULT_BUDGET };
+  for (const [rawKey, rawValue] of Object.entries(input as Record<string, unknown>)) {
+    const key = KEY_ALIASES[normalizeKey(rawKey)];
+    if (!key) {
+      throw new Error(
+        `Unknown budget key "${rawKey}" in ${source}. Known keys: ${Object.keys(DEFAULT_BUDGET).join(', ')}`,
+      );
+    }
+    if (rawValue === null) {
+      budget[key] = null;
+      continue;
+    }
+    if (typeof rawValue !== 'number' || !Number.isFinite(rawValue)) {
+      throw new Error(
+        `Budget "${rawKey}" must be a number or null, got ${JSON.stringify(rawValue)}`,
+      );
+    }
+    if (rawValue < 0) {
+      throw new Error(`Budget "${rawKey}" must not be negative, got ${rawValue}`);
+    }
+    budget[key] = rawValue;
+  }
+  return budget;
+}
+
+/**
+ * Resolve `--budget`. The value is either inline JSON (starts with `{`) or a
+ * path to a JSON file. Omitted means DEFAULT_BUDGET.
+ */
+export async function loadBudget(spec?: string): Promise<Budget> {
+  if (spec === undefined || spec === '') return { ...DEFAULT_BUDGET };
+
+  const trimmed = spec.trim();
+  if (trimmed.startsWith('{')) {
+    let parsed: unknown;
+    try {
+      parsed = JSON.parse(trimmed);
+    } catch (err) {
+      throw new Error(`--budget is not valid JSON: ${(err as Error).message}`);
+    }
+    return parseBudget(parsed, 'inline --budget');
+  }
+
+  let text: string;
+  try {
+    text = await readFile(trimmed, 'utf8');
+  } catch {
+    throw new Error(`Budget file not found or unreadable: ${trimmed}`);
+  }
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(text);
+  } catch (err) {
+    throw new Error(`Budget file ${trimmed} is not valid JSON: ${(err as Error).message}`);
+  }
+  return parseBudget(parsed, trimmed);
+}

package/src/report/collect-a11y.ts

@@ -0,0 +1,200 @@
+/**
+ * The browser-facing half of the accessibility check: fetch the AX tree,
+ * sweep the DOM for tabindex (which the AX tree does not expose at all), and
+ * turn `ax-node:<id>` placeholders into real CSS selectors a human or an
+ * agent can act on. The rules themselves live in a11y.ts and stay pure.
+ */
+
+import { evaluateJs } from '../browser/actions.js';
+import type { CdpClient } from '../browser/cdp.js';
+import { runA11yRules } from './a11y.js';
+import { buildStructure } from './structure.js';
+import type { A11yFinding, AxNode, FocusCandidate, StructureNode } from './types.js';
+import { message, STEP_TIMEOUT_MS, withTimeout } from './util.js';
+
+/** Most AX findings we will spend a DOM round-trip on to get a real selector. */
+const MAX_LOCATOR_LOOKUPS = 60;
+
+/** Builds a short, stable CSS selector for an element, from inside the page. */
+const LOCATOR_FN = `function(){
+  var el = this;
+  if (!el || el.nodeType !== 1) return '';
+  var esc = function(s){ return window.CSS && CSS.escape ? CSS.escape(s) : s; };
+  if (el.id) return '#' + esc(el.id);
+  var parts = [], node = el, depth = 0;
+  while (node && node.nodeType === 1 && depth < 5) {
+    var part = node.tagName.toLowerCase();
+    if (node.id) { parts.unshift('#' + esc(node.id)); break; }
+    var parent = node.parentElement;
+    if (parent) {
+      var sibs = Array.prototype.filter.call(parent.children, function(c){ return c.tagName === node.tagName; });
+      if (sibs.length > 1) part += ':nth-of-type(' + (sibs.indexOf(node) + 1) + ')';
+    }
+    parts.unshift(part);
+    node = parent;
+    depth++;
+  }
+  return parts.join(' > ');
+}`;
+
+/** DOM sweep for tabindex — the AX tree does not expose it at all. */
+const FOCUS_CANDIDATES_JS = `(function(){
+  try {
+    var esc = function(s){ return window.CSS && CSS.escape ? CSS.escape(s) : s; };
+    var loc = function(el){
+      if (el.id) return '#' + esc(el.id);
+      var parts = [], node = el, depth = 0;
+      while (node && node.nodeType === 1 && depth < 5) {
+        var part = node.tagName.toLowerCase();
+        if (node.id) { parts.unshift('#' + esc(node.id)); break; }
+        var parent = node.parentElement;
+        if (parent) {
+          var sibs = Array.prototype.filter.call(parent.children, function(c){ return c.tagName === node.tagName; });
+          if (sibs.length > 1) part += ':nth-of-type(' + (sibs.indexOf(node) + 1) + ')';
+        }
+        parts.unshift(part);
+        node = parent;
+        depth++;
+      }
+      return parts.join(' > ');
+    };
+    var out = [];
+    var els = document.querySelectorAll('[tabindex]');
+    for (var i = 0; i < els.length && out.length < 100; i++) {
+      var el = els[i];
+      var ti = parseInt(el.getAttribute('tabindex'), 10);
+      if (!(ti < 0)) continue;
+      var name = (el.getAttribute('aria-label') || el.getAttribute('title') || (el.textContent || '')).trim();
+      out.push({
+        tag: el.tagName.toLowerCase(),
+        role: (el.getAttribute('role') || '').toLowerCase(),
+        name: name ? name.slice(0, 120) : null,
+        tabindex: ti,
+        locator: loc(el)
+      });
+    }
+    return out;
+  } catch (e) { return []; }
+})()`;
+
+async function resolveLocator(
+  client: CdpClient,
+  sessionId: string,
+  backendNodeId: number,
+): Promise<string | null> {
+  try {
+    const resolved = await client.send<{ object?: { objectId?: string } }>(
+      'DOM.resolveNode',
+      { backendNodeId },
+      sessionId,
+    );
+    const objectId = resolved.object?.objectId;
+    if (!objectId) return null;
+    try {
+      const call = await client.send<{ result?: { value?: string } }>(
+        'Runtime.callFunctionOn',
+        { functionDeclaration: LOCATOR_FN, objectId, returnByValue: true },
+        sessionId,
+      );
+      return call.result?.value || null;
+    } finally {
+      await client.send('Runtime.releaseObject', { objectId }, sessionId).catch(() => {});
+    }
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Rewrites `ax-node:<id>` placeholder locators into real CSS selectors.
+ * Bounded: a page with 400 unlabelled buttons must not cost 400 CDP
+ * round-trips, and nobody reads the 400th finding anyway.
+ */
+export async function upgradeLocators(
+  client: CdpClient,
+  sessionId: string,
+  findings: A11yFinding[],
+  nodes: AxNode[],
+): Promise<A11yFinding[]> {
+  const backendByNodeId = new Map<string, number>();
+  for (const n of nodes) {
+    if (n.backendDOMNodeId !== undefined) backendByNodeId.set(String(n.nodeId), n.backendDOMNodeId);
+  }
+
+  let budget = MAX_LOCATOR_LOOKUPS;
+  const cache = new Map<number, string | null>();
+  const out: A11yFinding[] = [];
+  for (const finding of findings) {
+    if (!finding.locator.startsWith('ax-node:') || budget <= 0) {
+      out.push(finding);
+      continue;
+    }
+    const backendId = backendByNodeId.get(finding.locator.slice('ax-node:'.length));
+    if (backendId === undefined) {
+      out.push(finding);
+      continue;
+    }
+    if (!cache.has(backendId)) {
+      budget--;
+      cache.set(backendId, await resolveLocator(client, sessionId, backendId));
+    }
+    const selector = cache.get(backendId);
+    out.push(selector ? { ...finding, locator: selector } : finding);
+  }
+  return out;
+}
+
+export interface A11yResult {
+  findings: A11yFinding[];
+  /**
+   * Flat signature of the same tree, for the run-to-run structural diff. It
+   * rides along here because the AX tree is already in hand — fetching it
+   * twice would double the cost of the most expensive collector.
+   */
+  structure: StructureNode[];
+}
+
+export async function collectA11y(
+  client: CdpClient,
+  sessionId: string,
+  notes: string[],
+): Promise<A11yResult> {
+  let nodes: AxNode[] = [];
+  try {
+    const tree = await withTimeout(
+      client.send<{ nodes: AxNode[] }>('Accessibility.getFullAXTree', {}, sessionId),
+      STEP_TIMEOUT_MS,
+      'accessibility tree',
+    );
+    nodes = tree.nodes ?? [];
+  } catch (err) {
+    notes.push(`Accessibility tree unavailable: ${message(err)}`);
+    return { findings: [], structure: [] };
+  }
+
+  let focusCandidates: FocusCandidate[] = [];
+  try {
+    const raw = await withTimeout(
+      evaluateJs(client, sessionId, FOCUS_CANDIDATES_JS),
+      STEP_TIMEOUT_MS,
+      'tabindex sweep',
+    );
+    if (Array.isArray(raw)) focusCandidates = raw as FocusCandidate[];
+  } catch (err) {
+    notes.push(`Tabindex sweep skipped: ${message(err)}`);
+  }
+
+  const structure = buildStructure(nodes);
+  const findings = runA11yRules({ nodes, focusCandidates });
+  try {
+    const upgraded = await withTimeout(
+      upgradeLocators(client, sessionId, findings, nodes),
+      STEP_TIMEOUT_MS,
+      'locator resolution',
+    );
+    return { findings: upgraded, structure };
+  } catch {
+    notes.push('Some accessibility locators could not be resolved to CSS selectors.');
+    return { findings, structure };
+  }
+}