@sdods/core 0.8.0 → 0.10.0

package/dist/perf/vitals.js

@@ -0,0 +1,236 @@
+import { SdodsError } from '../errors.js';
+/**
+ * Page vitals, latency percentiles and the budgets they are judged against, shared by the explicit
+ * perf steps (`steps/perf.steps.ts`) and the end-of-scenario check every `@perf` scenario gets
+ * (`quality/hooks.ts`). Outside the step file for the reason `a11y/audit.ts` gives: importing a step
+ * file registers its phrases, and `steps.core.exclude: [perf]` must stay able to switch them off.
+ * The rules these functions keep are written up at the top of `steps/perf.steps.ts`.
+ */
+/* ── budgets ──────────────────────────────────────────────────────────── */
+/** The page vitals SDODS records. Each name is also its key in `perf.budgets`. */
+export const VITAL_KEYS = ['pageLoadMs', 'lcpMs', 'fcpMs', 'ttfbMs'];
+/** The `perf.budgets` key a sampled API latency distribution is judged against. */
+export const API_P95_BUDGET_KEY = 'apiP95Ms';
+/**
+ * Effective budgets, env overriding project per key.
+ *
+ * `resolveConfig` already deep-merges `env.perf` into `config.project.perf`, so on a resolved
+ * config the project side is the whole answer. Merging the env side again is idempotent and keeps
+ * this correct for a config assembled by hand. Non-numeric values are dropped rather than spread —
+ * an explicit `undefined` in the env layer would otherwise erase a real project budget and turn a
+ * configured assertion into an unconfigured one.
+ */
+export function resolvedBudgets(config) {
+    const out = {};
+    for (const layer of [config.project.perf?.budgets, config.env.perf?.budgets]) {
+        for (const [key, value] of Object.entries(layer ?? {})) {
+            if (typeof value === 'number' && Number.isFinite(value))
+                out[key] = value;
+        }
+    }
+    return out;
+}
+/**
+ * The configured budget for one key, or a hard error naming it.
+ *
+ * PROVES the assertion that follows is gated on something real. A missing budget must never
+ * degrade to a skip or to `Infinity`: an unconfigured budget is the single failure mode that lets
+ * an entire perf suite report success while asserting nothing at all.
+ */
+export function requireBudget(config, key) {
+    const budget = resolvedBudgets(config)[key];
+    const envName = config.env.name ?? 'unknown';
+    if (typeof budget !== 'number' || !(budget > 0)) {
+        throw new SdodsError('CONFIG_INVALID', `Perf budget "${key}" is not configured for environment "${envName}".`, {
+            hint: `Set perf.budgets.${key} in the project's sdods.project.yaml, or override it in envs/${envName}.yaml. Without it this assertion would pass no matter how slow the application is.`,
+        });
+    }
+    return budget;
+}
+export function requireVitalKey(name) {
+    if (VITAL_KEYS.includes(name))
+        return name;
+    throw new SdodsError('CONFIG_INVALID', `"${name}" is not a page vital SDODS records.`, {
+        hint: `Known vitals: ${VITAL_KEYS.join(', ')}. Each is also its key in perf.budgets.`,
+    });
+}
+/**
+ * Measurements have to survive from a `When` step to a `Then` step. They do not go in
+ * `apiContext.vars`: that store doubles as a `{{…}}` template scope, so an array of samples parked
+ * there would leak into every later rendering. Keyed on the per-test `scenario` object, so two
+ * scenarios in parallel workers can never read each other's numbers and nothing outlives the
+ * scenario that measured it.
+ */
+const stores = new WeakMap();
+export function perfStore(key) {
+    let store = stores.get(key);
+    if (!store) {
+        store = { recordings: [], sampleSets: [] };
+        stores.set(key, store);
+    }
+    return store;
+}
+export function lastRecording(key) {
+    const recording = perfStore(key).recordings.at(-1);
+    if (!recording) {
+        throw new SdodsError('RUN_FAILED', 'No page vitals have been recorded in this scenario.', {
+            hint: 'Add `When I record the page vitals` after the navigation you want to measure.',
+        });
+    }
+    return recording;
+}
+export function lastSampleSet(key) {
+    const set = perfStore(key).sampleSets.at(-1);
+    if (!set) {
+        throw new SdodsError('RUN_FAILED', 'No latency samples have been taken in this scenario.', {
+            hint: 'Add `When I sample the latency of GET "/path" over 20 requests` before asserting a p95.',
+        });
+    }
+    return set;
+}
+/* ── statistics ───────────────────────────────────────────────────────── */
+/**
+ * Nearest-rank p95 — deterministic for a given set of samples, with no interpolation to argue
+ * about. Below 20 samples the nearest rank IS the maximum; the sample count travels in the
+ * attachment and in every failure message so a reader can see when "p95" means "slowest of five".
+ *
+ * Throws rather than returning 0 on an empty sample: a p95 of 0 slides under every budget.
+ */
+export function p95Of(msValues) {
+    if (msValues.length === 0) {
+        throw new SdodsError('RUN_FAILED', 'Cannot compute a p95 over an empty sample.', {
+            hint: 'Sample at least one request before asserting a p95.',
+        });
+    }
+    const sorted = [...msValues].sort((a, b) => a - b);
+    const index = Math.max(0, Math.min(sorted.length - 1, Math.ceil(sorted.length * 0.95) - 1));
+    return sorted[index];
+}
+/* ── browser-side collection ──────────────────────────────────────────── */
+/**
+ * How long the LCP observer is given to settle. LCP arrives as a stream of ever-larger candidates;
+ * `buffered: true` replays the ones already emitted and this window catches any still in flight.
+ * A fixed constant rather than a computed wait, so two runs of one scenario observe the same
+ * window and a baseline stays comparable.
+ */
+export const LCP_SETTLE_MS = 500;
+/** Bounded wait for the load event before reading `loadEventEnd`. */
+export const LOAD_WAIT_MS = 5_000;
+/**
+ * Runs inside the page. Serialised by Playwright, so it closes over nothing but its argument — the
+ * type aliases above are erased at compile time and are safe to reference; the constants are not,
+ * which is why the settle window is passed in.
+ */
+export async function collectRawVitals(settleMs) {
+    const perf = performance;
+    const Observer = globalThis
+        .PerformanceObserver;
+    // Every vital is an offset from navigation start, so a genuine measurement is always > 0.
+    const positive = (value) => typeof value === 'number' && Number.isFinite(value) && value > 0 ? Math.round(value) : null;
+    const supported = Observer?.supportedEntryTypes ?? [];
+    const lcpSupported = supported.includes('largest-contentful-paint');
+    const fcpSupported = supported.includes('paint');
+    // TRAP: paint timings can land AFTER the load event. Reading the paint entry eagerly while only
+    // LCP waited for the settle window reported `fcpMs: null` beside a real `lcpMs` on a page that
+    // had painted perfectly well — impossible, since first paint always precedes the largest one —
+    // and a budget assertion would then have failed it as "not measured". So nothing is read until
+    // the window closes, and the window is opened whether or not LCP is supported.
+    let largestPaint = 0;
+    await new Promise((resolve) => {
+        let observer;
+        try {
+            if (Observer && lcpSupported) {
+                observer = new Observer((list) => {
+                    for (const entry of list.getEntries()) {
+                        const start = entry.startTime;
+                        if (typeof start === 'number' && start > largestPaint)
+                            largestPaint = start;
+                    }
+                });
+                observer.observe({ type: 'largest-contentful-paint', buffered: true });
+            }
+        }
+        catch {
+            observer = undefined;
+        }
+        setTimeout(() => {
+            try {
+                observer?.disconnect();
+            }
+            catch {
+                /* already gone */
+            }
+            resolve();
+        }, settleMs);
+    });
+    const nav = perf.getEntriesByType('navigation')[0];
+    const fcpEntry = perf.getEntriesByName('first-contentful-paint')[0];
+    const lcpMs = largestPaint > 0 ? Math.round(largestPaint) : null;
+    let totalResources = 0;
+    let totalBytes = 0;
+    for (const entry of perf.getEntriesByType('resource')) {
+        totalResources++;
+        const size = entry.transferSize;
+        if (typeof size === 'number' && Number.isFinite(size))
+            totalBytes += size;
+    }
+    const startTime = nav && typeof nav.startTime === 'number' ? nav.startTime : 0;
+    return {
+        hasNavigationEntry: Boolean(nav),
+        navigationStartEpochMs: nav && typeof perf.timeOrigin === 'number' ? Math.round(perf.timeOrigin + startTime) : null,
+        pageLoadMs: nav ? positive(nav.loadEventEnd) : null,
+        ttfbMs: nav ? positive(nav.responseStart) : null,
+        fcpMs: fcpEntry ? positive(fcpEntry.startTime) : null,
+        lcpMs,
+        domContentLoadedMs: nav ? positive(nav.domContentLoadedEventEnd) : null,
+        totalResources,
+        totalResourceSizeKB: Math.round(totalBytes / 1024),
+        lcpSupported,
+        fcpSupported,
+    };
+}
+/**
+ * Turns the page's answer into a recording, capturing WHY a vital is null while the browser's
+ * capabilities are still in hand. By the time a `Then` step fails, "lcpMs was not measured" on its
+ * own sends the reader hunting for a bug in a page that is merely running in a browser that does
+ * not implement LCP (WebKit). Support is read from `PerformanceObserver.supportedEntryTypes`, never
+ * assumed from the browser name.
+ */
+export function toRecording(raw, url, browserName) {
+    const vitals = {
+        pageLoadMs: raw.pageLoadMs,
+        lcpMs: raw.lcpMs,
+        fcpMs: raw.fcpMs,
+        ttfbMs: raw.ttfbMs,
+    };
+    const where = browserName ? ` in ${browserName}` : ' in this browser';
+    const unmeasured = {};
+    if (raw.pageLoadMs === null)
+        unmeasured.pageLoadMs = `the load event had not fired ${LOAD_WAIT_MS} ms after the vitals step began`;
+    if (raw.ttfbMs === null)
+        unmeasured.ttfbMs = 'the navigation entry reported no response start';
+    if (raw.fcpMs === null)
+        unmeasured.fcpMs = raw.fcpSupported
+            ? 'the page painted no contentful frame'
+            : `paint timings are not implemented${where}`;
+    if (raw.lcpMs === null)
+        unmeasured.lcpMs = raw.lcpSupported
+            ? 'the page produced no largest-contentful-paint candidate'
+            : `largest-contentful-paint is not implemented${where} — restrict the scenario to a browser that supports it`;
+    // The contract's three non-nullable fields carry 0 for an unmeasured value; the nullable
+    // `budgets.measured` block written alongside is what the assertions read, so a 0 here can never
+    // be mistaken for a measurement.
+    const metrics = {
+        url,
+        timestamp: new Date(raw.navigationStartEpochMs ?? Date.now()).toISOString(),
+        domContentLoaded: raw.domContentLoadedMs ?? 0,
+        pageLoadTime: raw.pageLoadMs ?? 0,
+        timeToFirstByte: raw.ttfbMs ?? 0,
+        totalResources: raw.totalResources,
+        totalResourceSizeKB: raw.totalResourceSizeKB,
+        firstContentfulPaint: raw.fcpMs,
+        largestContentfulPaint: raw.lcpMs,
+    };
+    return { url, vitals, unmeasured, metrics };
+}
+//# sourceMappingURL=vitals.js.map

package/dist/quality/a11y-scenario.d.ts

@@ -0,0 +1,110 @@
+import { type Page } from '@playwright/test';
+import { type A11yConfig } from '@sdods/contracts';
+import { type A11yResults } from '../a11y/audit.js';
+/**
+ * The audit an `@a11y` tag buys. Before this existed the tag was a label: the linter accepted it,
+ * the docs said it ran axe, and nothing read it.
+ *
+ * At the END of an `@a11y` UI or hybrid scenario, the page the scenario finished on is audited with
+ * axe (WCAG 2.x A/AA, as the explicit steps are) and the scenario fails on any violation at or above
+ * `a11y.failOn`. The report is attached as `sdods/a11y-scenario` pass or fail, and it is also what
+ * the `a11y` process gate reads.
+ *
+ * `a11y.scope: every-page` widens that to every distinct URL the scenario reaches. Each one is
+ * audited after the STEP that reached it, once the page has settled at the point the `@perf` hook
+ * reads a document (load, first paint, the LCP settle window). The trigger is the step boundary
+ * rather than the page's own load event on purpose: an audit started from a load listener runs
+ * beside the next step's clicks, and a step that leaves the page mid-audit destroys the context axe
+ * runs in, so whether a page got audited would depend on timing. Between two steps nothing else
+ * drives the page. Violations are collected per URL and fail the scenario once, at its end, naming
+ * every URL that has them. A URL the scenario passed through inside a single step (a redirect, or a
+ * step that clicks through two pages) is listed as `not-audited`, so the gap is visible.
+ *
+ * What it deliberately does not do:
+ *   · audit a scenario that already failed. The page it stopped on is wherever the failure left it,
+ *     and a violation list on top of the real error only buries the error;
+ *   · audit a URL an explicit whole-page audit step already covered in the same scenario. That step
+ *     ran with the threshold its author chose, and it passed (a failed step would have failed the
+ *     scenario). Under `every-page` this holds even when the step ran after the tag's own audit;
+ *   · with the default `scope: final`, audit any page but the last one.
+ */
+export type A11yScenarioStatus = 'audited' | 'covered-by-step' | 'skipped';
+/** One URL's audit under `a11y.scope: every-page`. */
+export interface A11yPageReport {
+    url: string;
+    status: 'audited' | 'covered-by-step' | 'not-audited';
+    reason?: string;
+    /** When the page was audited: after the named step, or at the end of the scenario. */
+    auditedAfter?: string;
+    violations: number;
+    blocking: number;
+    blockingRules: string[];
+    results?: A11yResults;
+}
+/** What `sdods/a11y-scenario` contains, and what the `a11y` process gate reads back. */
+export interface A11yScenarioReport {
+    kind: 'a11y-scenario';
+    runnerProject: string;
+    fingerprint: string;
+    retry: number;
+    /** The URL the scenario ended on. */
+    url: string;
+    status: A11yScenarioStatus;
+    reason?: string;
+    /** Absent in reports written before `a11y.scope` existed, which were all `final`. */
+    scope?: A11yConfig['scope'];
+    failOn: A11yConfig['failOn'];
+    include: string[];
+    exclude: string[];
+    /** Every violation axe reported, at any impact (summed over `pages` under `every-page`). */
+    violations: number;
+    /** Violations at or above `failOn`: the ones that fail the scenario. */
+    blocking: number;
+    blockingRules: string[];
+    /** The axe results of the final page (`final` scope only; `every-page` keeps them per page). */
+    results?: A11yResults;
+    /** Every URL the scenario reached, in order (`every-page` scope only). */
+    pages?: A11yPageReport[];
+}
+export interface A11yScenarioDeps {
+    page: Page;
+    config: {
+        project: {
+            a11y?: Partial<A11yConfig>;
+        };
+    };
+    scenario: {
+        data: {
+            runnerProject: string;
+            fingerprint: string;
+            retry: number;
+        };
+        file(rel: string): string;
+    };
+    testInfo: {
+        status?: string;
+        attach(name: string, opts: {
+            path: string;
+            contentType: string;
+        }): Promise<void>;
+    };
+}
+/** Effective a11y settings with the schema defaults, so a hand-built config behaves like a parsed one. */
+export declare function a11ySettings(config: A11yScenarioDeps['config']): A11yConfig;
+/**
+ * Under `every-page`, starts listing the URLs the page's main frame reaches, so a URL no step ended
+ * on still shows up in the report. A no-op under `final`.
+ */
+export declare function watchA11yPages(deps: Pick<A11yScenarioDeps, 'page' | 'config' | 'scenario'>): void;
+/**
+ * Under `every-page`, audits the page after a step when the step left it on a URL not yet audited
+ * or covered. Violations are kept for the end of the scenario; a configuration defect (an `include`
+ * that matches nothing, an audit that examined nothing) fails the step it follows.
+ */
+export declare function auditAfterStep(deps: Pick<A11yScenarioDeps, 'page' | 'config' | 'scenario'> & {
+    step: string;
+    /** True when the step failed: its page proves nothing, as at the end of a failed scenario. */
+    failed?: boolean;
+}): Promise<A11yPageReport | undefined>;
+export declare function auditScenarioEnd(deps: A11yScenarioDeps): Promise<A11yScenarioReport>;
+//# sourceMappingURL=a11y-scenario.d.ts.map

package/dist/quality/a11y-scenario.js

@@ -0,0 +1,229 @@
+import { expect } from '@playwright/test';
+import { attachmentNames, scenarioFiles } from '@sdods/contracts';
+import { IMPACT_ORDER, assertAxeChecked, describeViolations, parseImpactFloor, runAxe, violationsAtOrAbove, wasPageAudited, writeAndAttachJson, } from '../a11y/audit.js';
+import { SdodsError } from '../errors.js';
+import { waitForSettle } from './perf-scenario.js';
+/** Effective a11y settings with the schema defaults, so a hand-built config behaves like a parsed one. */
+export function a11ySettings(config) {
+    const a = config.project.a11y ?? {};
+    return {
+        failOn: a.failOn ?? 'serious',
+        include: a.include ?? [],
+        exclude: a.exclude ?? [],
+        scope: a.scope ?? 'final',
+    };
+}
+const pageWatches = new WeakMap();
+function watchOf(scenario) {
+    let watch = pageWatches.get(scenario);
+    if (!watch)
+        pageWatches.set(scenario, (watch = { visited: [], pages: new Map() }));
+    return watch;
+}
+const auditable = (url) => Boolean(url) && url !== 'about:blank';
+/**
+ * Under `every-page`, starts listing the URLs the page's main frame reaches, so a URL no step ended
+ * on still shows up in the report. A no-op under `final`.
+ */
+export function watchA11yPages(deps) {
+    if (a11ySettings(deps.config).scope !== 'every-page')
+        return;
+    const watch = watchOf(deps.scenario);
+    deps.page.on('framenavigated', (frame) => {
+        if (frame !== deps.page.mainFrame())
+            return;
+        const url = frame.url();
+        if (auditable(url) && !watch.visited.includes(url))
+            watch.visited.push(url);
+    });
+}
+/** A navigation that replaced the document axe was running in: not a defect of the page. */
+const LEFT_THE_PAGE = /execution context was destroyed|frame was detached|navigat/i;
+async function auditPage(page, settings, url, auditedAfter) {
+    const results = await runAxe(page, { include: settings.include, exclude: settings.exclude });
+    assertAxeChecked(results, `on ${url}`);
+    const blocking = violationsAtOrAbove(results.violations, parseImpactFloor(settings.failOn));
+    return {
+        url,
+        status: 'audited',
+        auditedAfter,
+        violations: results.violations.length,
+        blocking: blocking.length,
+        blockingRules: blocking.map((v) => v.id),
+        results,
+    };
+}
+/**
+ * Under `every-page`, audits the page after a step when the step left it on a URL not yet audited
+ * or covered. Violations are kept for the end of the scenario; a configuration defect (an `include`
+ * that matches nothing, an audit that examined nothing) fails the step it follows.
+ */
+export async function auditAfterStep(deps) {
+    const { page, scenario } = deps;
+    const settings = a11ySettings(deps.config);
+    if (settings.scope !== 'every-page' || deps.failed)
+        return undefined;
+    const watch = watchOf(scenario);
+    watch.fresh = undefined;
+    const before = page.url();
+    if (!auditable(before) || watch.pages.has(before))
+        return undefined;
+    if (wasPageAudited(scenario, before))
+        return covered(watch, before);
+    await waitForSettle(page);
+    // The page may have redirected while it settled; audit where it is now.
+    const url = page.url();
+    if (!auditable(url) || watch.pages.has(url))
+        return undefined;
+    if (wasPageAudited(scenario, url))
+        return covered(watch, url);
+    if (!watch.visited.includes(url))
+        watch.visited.push(url);
+    try {
+        const report = await auditPage(page, settings, url, deps.step);
+        watch.pages.set(url, report);
+        watch.fresh = url;
+        return report;
+    }
+    catch (e) {
+        // A client-side redirect after the settle point: the next step boundary, or the end of the
+        // scenario, audits wherever it went. This URL is listed as not audited.
+        if (e instanceof Error && LEFT_THE_PAGE.test(e.message) && page.url() !== url)
+            return undefined;
+        throw e;
+    }
+}
+function covered(watch, url) {
+    const report = {
+        url,
+        status: 'covered-by-step',
+        reason: 'an explicit whole-page accessibility step already audited this URL in the scenario',
+        violations: 0,
+        blocking: 0,
+        blockingRules: [],
+    };
+    if (!watch.visited.includes(url))
+        watch.visited.push(url);
+    watch.pages.set(url, report);
+    return report;
+}
+/* ── the end of the scenario ─────────────────────────────────────────────── */
+export async function auditScenarioEnd(deps) {
+    const { page, scenario, testInfo } = deps;
+    const settings = a11ySettings(deps.config);
+    const floor = parseImpactFloor(settings.failOn);
+    const url = page.url();
+    const everyPage = settings.scope === 'every-page';
+    const base = {
+        kind: 'a11y-scenario',
+        runnerProject: scenario.data.runnerProject,
+        fingerprint: scenario.data.fingerprint,
+        retry: scenario.data.retry,
+        url,
+        ...(everyPage ? { scope: settings.scope } : {}),
+        failOn: settings.failOn,
+        include: settings.include,
+        exclude: settings.exclude,
+        violations: 0,
+        blocking: 0,
+        blockingRules: [],
+    };
+    const file = scenario.file(scenarioFiles.a11yScenarioJson(scenario.data.runnerProject));
+    const write = (report) => writeAndAttachJson(testInfo, attachmentNames.a11yScenario, file, report).then(() => report);
+    if (testInfo.status && testInfo.status !== 'passed') {
+        return write({
+            ...base,
+            status: 'skipped',
+            reason: `the scenario ${testInfo.status} before the audit, so the page it stopped on proves nothing`,
+        });
+    }
+    const watch = everyPage ? watchOf(scenario) : undefined;
+    if (!auditable(url) && !watch?.pages.size) {
+        throw new SdodsError('RUN_FAILED', 'This @a11y scenario never navigated anywhere, so there is no page to audit.', {
+            hint: 'Navigate to the page under test in the scenario, or drop @a11y from a scenario that has no page.',
+        });
+    }
+    if (watch)
+        return auditEveryPageEnd({ ...deps, settings, url, watch, base, write });
+    if (wasPageAudited(scenario, url)) {
+        return write({
+            ...base,
+            status: 'covered-by-step',
+            reason: 'an explicit whole-page accessibility step already audited this URL in the scenario',
+        });
+    }
+    const results = await runAxe(page, { include: settings.include, exclude: settings.exclude });
+    const blocking = violationsAtOrAbove(results.violations, floor);
+    const report = await write({
+        ...base,
+        status: 'audited',
+        violations: results.violations.length,
+        blocking: blocking.length,
+        blockingRules: blocking.map((v) => v.id),
+        results,
+    });
+    assertAxeChecked(results, `on ${url}`);
+    expect(blocking.length, `@a11y: ${blocking.length} violation(s) at ${IMPACT_ORDER[floor]} or worse on ${url} (of ${results.violations.length} total; a11y.failOn is "${settings.failOn}"):\n${describeViolations(blocking)}`).toBe(0);
+    return report;
+}
+async function auditEveryPageEnd(input) {
+    const { page, scenario, settings, url, watch, base, write } = input;
+    const floor = parseImpactFloor(settings.failOn);
+    // The final page is audited as it stands at the end, as under `final`: the steps after the one
+    // that reached it may have changed what is on it. Unless the last step's own audit is that state.
+    if (auditable(url) && watch.fresh !== url) {
+        if (!watch.visited.includes(url))
+            watch.visited.push(url);
+        if (wasPageAudited(scenario, url))
+            covered(watch, url);
+        else
+            watch.pages.set(url, await auditPage(page, settings, url, 'the end of the scenario'));
+    }
+    const pages = watch.visited.map((visited) => {
+        const audit = watch.pages.get(visited);
+        // An explicit step that audited this URL AFTER the tag did has the last word, as it does under `final`.
+        if (audit?.status === 'audited' && wasPageAudited(scenario, visited))
+            return {
+                url: visited,
+                status: 'covered-by-step',
+                reason: 'an explicit whole-page accessibility step audited this URL later in the scenario, with the threshold its author chose',
+                violations: 0,
+                blocking: 0,
+                blockingRules: [],
+            };
+        return (audit ?? {
+            url: visited,
+            status: 'not-audited',
+            reason: 'the scenario left this URL within the step that reached it, so no step boundary found it',
+            violations: 0,
+            blocking: 0,
+            blockingRules: [],
+        });
+    });
+    const audited = pages.filter((p) => p.status === 'audited');
+    const failing = audited.filter((p) => p.blocking > 0);
+    const report = await write({
+        ...base,
+        status: audited.length ? 'audited' : 'covered-by-step',
+        ...(audited.length
+            ? {}
+            : {
+                reason: 'explicit whole-page accessibility steps covered every URL the scenario reached',
+            }),
+        violations: audited.reduce((n, p) => n + p.violations, 0),
+        blocking: audited.reduce((n, p) => n + p.blocking, 0),
+        blockingRules: [...new Set(failing.flatMap((p) => p.blockingRules))],
+        pages,
+    });
+    if (failing.length) {
+        const byUrl = failing
+            .map((p) => {
+            const blocking = violationsAtOrAbove(p.results?.violations ?? [], floor);
+            return `${p.url} — ${p.blocking} of ${p.violations}:\n${describeViolations(blocking)}`;
+        })
+            .join('\n');
+        expect(report.blocking, `@a11y: ${report.blocking} violation(s) at ${IMPACT_ORDER[floor]} or worse on ${failing.length} of ${audited.length} audited page(s) (a11y.scope is "every-page", a11y.failOn is "${settings.failOn}"):\n${byUrl}`).toBe(0);
+    }
+    return report;
+}
+//# sourceMappingURL=a11y-scenario.js.map

package/dist/quality/gates.d.ts

@@ -0,0 +1,87 @@
+import type { GateResult, GateRow, ProcessConfig, ProcessGates, RunManifest } from '@sdods/contracts';
+import type { A11yScenarioReport } from './a11y-scenario.js';
+import type { PerfScenarioReport } from './perf-scenario.js';
+export type { GateResult, GateRow };
+/**
+ * Process gates: `gates:` on a process in `sdods.project.yaml`.
+ *
+ * Until this existed the web UI saved `minPassRate`, `maxFlaky`, `a11y` and `perfBudgets`, the
+ * release process declared all four, and nothing anywhere evaluated them. A gate that is read by
+ * nothing is worse than no gate: it is the reason a release goes out believing it was checked.
+ *
+ * Evaluated by `sdods run --process <name>` from the run it just finished:
+ *   · `minPassRate` — (passed + flaky) / (total − skipped), as a percentage, the same arithmetic as
+ *     the run dashboard. A scenario that passed only on retry counts as passed here and as flaky
+ *     below. A run that executed nothing has no pass rate, and fails the gate.
+ *   · `maxFlaky` — scenarios that failed and then passed on a retry.
+ *   · `a11y` — every `@a11y` audit in the run found nothing at or above `a11y.failOn`, AND at least
+ *     one audit ran. Violations already fail their scenarios; what this gate adds is the proof that
+ *     the audits happened at all, because a process declaring `a11y: true` over a selection with no
+ *     `@a11y` scenario would otherwise pass having checked nothing.
+ *   · `perfBudgets` — the same for `@perf`: no breach in any judged scenario, and at least one
+ *     scenario was judged.
+ *
+ * The a11y and perf evidence is read from the scenario reports the tag hooks write into the run
+ * directory, taking the last attempt of each scenario on each runner project.
+ */
+export interface GateTotals {
+    total: number;
+    passed: number;
+    failed: number;
+    skipped: number;
+    flaky: number;
+}
+export interface GateEvidence {
+    a11y: A11yScenarioReport[];
+    perf: PerfScenarioReport[];
+}
+/** True when the process declares at least one gate worth evaluating. */
+export declare function hasGates(gates: Partial<ProcessGates> | undefined): boolean;
+/** Pure: the verdict for one process's gates over a run's totals and evidence. */
+export declare function evaluateGates(input: {
+    process: string;
+    gates: Partial<ProcessGates> | undefined;
+    totals: GateTotals | undefined;
+    evidence: GateEvidence;
+}): GateResult;
+/**
+ * Reads the `@a11y` and `@perf` scenario reports out of one or more run directories
+ * (`<runDir>/<slug>/<fingerprint>/r<retry>/{a11y,perf}/scenario--<runner project>.json`), keeping
+ * only the highest retry of each scenario on each runner project: an attempt that was retried is not
+ * the scenario's verdict. Several directories are how the shards of one run are read together: each
+ * shard wrote its own scenarios, and the same scenario never ran on two shards.
+ */
+export declare function collectGateEvidence(runDirs: string | readonly string[], 
+/** Reports read from elsewhere (the attachments of a merged report), judged the same way. */
+extra?: readonly (A11yScenarioReport | PerfScenarioReport)[]): GateEvidence;
+/**
+ * The `sdods/a11y-scenario` and `sdods/perf-scenario` reports attached to the tests of a runner JSON
+ * report. After `playwright merge-reports` their paths point into the directory the blob reports
+ * were unpacked into, which lives only as long as the merge, so read them before it is removed. A
+ * missing or unreadable file is skipped; the run directories are the other source of the same
+ * evidence.
+ */
+export declare function gateReportsFromRunnerJson(report: unknown): (A11yScenarioReport | PerfScenarioReport)[];
+/**
+ * Gate totals from a runner JSON report's `stats` (what `playwright merge-reports --reporter json`
+ * writes for the merged shards). The same counting as the run dashboard: one row per test, retries
+ * folded into `flaky`.
+ */
+export declare function gateTotalsFromRunnerStats(stats: {
+    expected?: number;
+    unexpected?: number;
+    flaky?: number;
+    skipped?: number;
+} | undefined): GateTotals | undefined;
+/**
+ * The gate judge ingest runs when a run directory carries no `gates.json`: the process the run's
+ * manifest names, over the totals ingest just computed and the evidence in the run directory.
+ * `undefined` when there is nothing it may judge: no process, a process without gates, a cancelled
+ * run, or one shard of several (a shard is not the run; `sdods report merge --process` judges it).
+ */
+export declare function ingestGateJudge(input: {
+    manifest: Partial<RunManifest> | null | undefined;
+    runDir: string;
+    processOf: (slug: string, name: string) => ProcessConfig;
+}): ((totals: GateTotals) => GateResult | undefined) | undefined;
+//# sourceMappingURL=gates.d.ts.map