@sdods/core 0.8.0 → 0.10.0

package/dist/quality/gates.js

@@ -0,0 +1,233 @@
+import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs';
+import { join } from 'node:path';
+import { attachmentNames } from '@sdods/contracts';
+/** True when the process declares at least one gate worth evaluating. */
+export function hasGates(gates) {
+    return Boolean(gates &&
+        (gates.minPassRate !== undefined ||
+            gates.maxFlaky !== undefined ||
+            gates.a11y === true ||
+            gates.perfBudgets === true));
+}
+/** Pure: the verdict for one process's gates over a run's totals and evidence. */
+export function evaluateGates(input) {
+    const { gates = {}, totals, evidence } = input;
+    const rows = [];
+    if (gates.minPassRate !== undefined) {
+        const executed = totals ? totals.total - totals.skipped : 0;
+        if (!totals || executed <= 0) {
+            rows.push({
+                gate: 'minPassRate',
+                threshold: `≥ ${gates.minPassRate}%`,
+                actual: 'n/a',
+                passed: false,
+                detail: totals
+                    ? 'no scenario executed, so there is no pass rate to meet the gate'
+                    : 'the run produced no results to compute a pass rate from',
+            });
+        }
+        else {
+            const rate = ((totals.passed + totals.flaky) / executed) * 100;
+            rows.push({
+                gate: 'minPassRate',
+                threshold: `≥ ${gates.minPassRate}%`,
+                actual: `${formatPct(rate)}% (${totals.passed + totals.flaky}/${executed})`,
+                passed: rate >= gates.minPassRate,
+            });
+        }
+    }
+    if (gates.maxFlaky !== undefined) {
+        rows.push({
+            gate: 'maxFlaky',
+            threshold: `≤ ${gates.maxFlaky}`,
+            actual: totals ? String(totals.flaky) : 'n/a',
+            passed: Boolean(totals) && (totals?.flaky ?? 0) <= gates.maxFlaky,
+            detail: totals ? undefined : 'the run produced no results to count flaky scenarios from',
+        });
+    }
+    if (gates.a11y) {
+        const reports = evidence.a11y.filter((r) => r.status !== 'skipped');
+        // Under `a11y.scope: every-page` one scenario report holds an audit per URL; each is counted.
+        const audits = reports.reduce((n, r) => n + auditsIn(r), 0);
+        const blocking = reports.reduce((n, r) => n + r.blocking, 0);
+        const failing = reports.flatMap((r) => (r.pages ?? [r])
+            .filter((p) => p.blocking > 0)
+            .map((p) => `${p.url} [${r.runnerProject}]: ${p.blockingRules.join(', ')}`));
+        rows.push({
+            gate: 'a11y',
+            threshold: 'no blocking violation',
+            actual: `${blocking} blocking in ${audits} audit(s)`,
+            passed: audits > 0 && blocking === 0,
+            detail: audits === 0
+                ? 'no @a11y audit ran in this run, so the gate has nothing to prove'
+                : failing.length
+                    ? failing.slice(0, 5).join('; ')
+                    : undefined,
+        });
+    }
+    if (gates.perfBudgets) {
+        const judged = evidence.perf.filter((r) => r.status === 'judged' && r.judged > 0);
+        const breaches = judged.flatMap((r) => r.breaches.map((b) => ({ ...b, runnerProject: r.runnerProject })));
+        rows.push({
+            gate: 'perfBudgets',
+            threshold: 'no budget breach',
+            actual: `${breaches.length} breach(es) in ${judged.length} scenario(s)`,
+            passed: judged.length > 0 && breaches.length === 0,
+            detail: judged.length === 0
+                ? 'no @perf scenario was judged against a budget in this run, so the gate has nothing to prove'
+                : breaches.length
+                    ? breaches
+                        .slice(0, 5)
+                        .map((b) => `${b.budget} ${b.measuredMs ?? 'unmeasured'}/${b.limitMs} ms at ${b.where}`)
+                        .join('; ')
+                    : undefined,
+        });
+    }
+    return { process: input.process, passed: rows.every((r) => r.passed), rows };
+}
+/** Audits one scenario report stands for: one per URL under `every-page`, otherwise one. */
+function auditsIn(report) {
+    if (!report.pages)
+        return 1;
+    return report.pages.filter((p) => p.status !== 'not-audited').length;
+}
+function formatPct(n) {
+    return Number.isInteger(n) ? String(n) : n.toFixed(1);
+}
+/**
+ * Reads the `@a11y` and `@perf` scenario reports out of one or more run directories
+ * (`<runDir>/<slug>/<fingerprint>/r<retry>/{a11y,perf}/scenario--<runner project>.json`), keeping
+ * only the highest retry of each scenario on each runner project: an attempt that was retried is not
+ * the scenario's verdict. Several directories are how the shards of one run are read together: each
+ * shard wrote its own scenarios, and the same scenario never ran on two shards.
+ */
+export function collectGateEvidence(runDirs, 
+/** Reports read from elsewhere (the attachments of a merged report), judged the same way. */
+extra = []) {
+    const latest = new Map();
+    const keep = (report) => {
+        const key = `${report.kind}|${report.fingerprint}|${report.runnerProject}`;
+        const prev = latest.get(key);
+        if (prev && prev.retry > report.retry)
+            return;
+        latest.set(key, report.kind === 'a11y-scenario'
+            ? { retry: report.retry, a11y: report }
+            : { retry: report.retry, perf: report });
+    };
+    const dirs = [...new Set(typeof runDirs === 'string' ? [runDirs] : runDirs)];
+    for (const file of dirs.flatMap(scenarioReportFiles)) {
+        try {
+            const parsed = JSON.parse(readFileSync(file, 'utf8'));
+            if (parsed.kind === 'a11y-scenario' || parsed.kind === 'perf-scenario')
+                keep(parsed);
+        }
+        catch {
+            /* a half-written report is not evidence */
+        }
+    }
+    for (const report of extra)
+        keep(report);
+    const values = [...latest.values()];
+    return {
+        a11y: values.flatMap((v) => (v.a11y ? [v.a11y] : [])),
+        perf: values.flatMap((v) => (v.perf ? [v.perf] : [])),
+    };
+}
+/**
+ * The `sdods/a11y-scenario` and `sdods/perf-scenario` reports attached to the tests of a runner JSON
+ * report. After `playwright merge-reports` their paths point into the directory the blob reports
+ * were unpacked into, which lives only as long as the merge, so read them before it is removed. A
+ * missing or unreadable file is skipped; the run directories are the other source of the same
+ * evidence.
+ */
+export function gateReportsFromRunnerJson(report) {
+    const names = new Set([attachmentNames.a11yScenario, attachmentNames.perfScenario]);
+    const out = [];
+    const walk = (suite) => {
+        for (const spec of suite.specs ?? [])
+            for (const test of spec.tests ?? [])
+                for (const result of test.results ?? [])
+                    for (const a of result.attachments ?? []) {
+                        if (!names.has(a.name) || !a.path || !existsSync(a.path))
+                            continue;
+                        try {
+                            const parsed = JSON.parse(readFileSync(a.path, 'utf8'));
+                            if (parsed.kind === 'a11y-scenario' || parsed.kind === 'perf-scenario')
+                                out.push(parsed);
+                        }
+                        catch {
+                            /* not evidence */
+                        }
+                    }
+        for (const child of suite.suites ?? [])
+            walk(child);
+    };
+    for (const suite of report?.suites ?? [])
+        walk(suite);
+    return out;
+}
+/**
+ * Gate totals from a runner JSON report's `stats` (what `playwright merge-reports --reporter json`
+ * writes for the merged shards). The same counting as the run dashboard: one row per test, retries
+ * folded into `flaky`.
+ */
+export function gateTotalsFromRunnerStats(stats) {
+    if (!stats)
+        return undefined;
+    const passed = stats.expected ?? 0;
+    const failed = stats.unexpected ?? 0;
+    const flaky = stats.flaky ?? 0;
+    const skipped = stats.skipped ?? 0;
+    return { total: passed + failed + flaky + skipped, passed, failed, skipped, flaky };
+}
+function scenarioReportFiles(runDir) {
+    const out = [];
+    const dirs = (p) => existsSync(p) ? readdirSync(p).filter((n) => statSync(join(p, n)).isDirectory()) : [];
+    for (const slug of dirs(runDir)) {
+        // Fingerprints are 16 hex characters; this keeps the walk out of the report directories.
+        for (const fp of dirs(join(runDir, slug)).filter((n) => /^[0-9a-f]{16}$/.test(n))) {
+            for (const attempt of dirs(join(runDir, slug, fp))) {
+                if (!/^r\d+$/.test(attempt))
+                    continue;
+                for (const kind of ['a11y', 'perf']) {
+                    const dir = join(runDir, slug, fp, attempt, kind);
+                    if (!existsSync(dir))
+                        continue;
+                    for (const name of readdirSync(dir))
+                        if (name.startsWith('scenario--') && name.endsWith('.json'))
+                            out.push(join(dir, name));
+                }
+            }
+        }
+    }
+    return out;
+}
+/**
+ * The gate judge ingest runs when a run directory carries no `gates.json`: the process the run's
+ * manifest names, over the totals ingest just computed and the evidence in the run directory.
+ * `undefined` when there is nothing it may judge: no process, a process without gates, a cancelled
+ * run, or one shard of several (a shard is not the run; `sdods report merge --process` judges it).
+ */
+export function ingestGateJudge(input) {
+    const { manifest, runDir } = input;
+    if (!manifest?.process || !manifest.projectSlug)
+        return undefined;
+    if ((manifest.shardTotal ?? 1) > 1 || manifest.exitCode === 130)
+        return undefined;
+    let proc;
+    try {
+        proc = input.processOf(manifest.projectSlug, manifest.process);
+    }
+    catch {
+        return undefined; // a process this checkout no longer defines has no gates to judge
+    }
+    if (!hasGates(proc.gates))
+        return undefined;
+    return (totals) => evaluateGates({
+        process: proc.name,
+        gates: proc.gates,
+        totals,
+        evidence: collectGateEvidence(runDir),
+    });
+}
+//# sourceMappingURL=gates.js.map

package/dist/quality/hooks.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=hooks.d.ts.map

package/dist/quality/hooks.js

@@ -0,0 +1,51 @@
+import { AfterScenario, AfterStep, BeforeScenario } from '../fixtures/test.js';
+import { auditAfterStep, auditScenarioEnd, watchA11yPages } from './a11y-scenario.js';
+import { judgeScenarioPerf, watchNavigations } from './perf-scenario.js';
+/**
+ * `@a11y` and `@perf` tag hooks.
+ *
+ * Tag-filtered scenario hooks rather than auto fixtures, for the reason `shots/hooks.ts` gives: a
+ * hook's fixtures are requested only by the scenarios its tag expression matches (playwright-bdd
+ * decides that when it generates the spec), so an `@api @perf` scenario never opens a browser to be
+ * judged. playwright-bdd reads the fixture names from the destructuring below, so each hook lists
+ * its fixtures literally and hands them to a plain function the unit tests call directly.
+ *
+ * Registered after the screenshot hooks and before nothing else: after-hooks run in reverse
+ * registration order, so these run BEFORE `sdods:finalize` and can leave their verdict on the
+ * scenario for it to record. A failure here is otherwise invisible to `meta.json`, because
+ * Playwright only marks the test failed once every after-hook has returned.
+ */
+const PAGE = '(@ui or @hybrid)';
+async function noting(scenario, run) {
+    try {
+        return await run();
+    }
+    catch (e) {
+        scenario.noteFailure(e instanceof Error ? e.message : String(e));
+        throw e;
+    }
+}
+BeforeScenario({ name: 'sdods:perf:watch', tags: `@perf and ${PAGE}` }, async ({ page, scenario }) => {
+    await watchNavigations({ page, scenario });
+});
+// Registered before the a11y hook, so it runs after it: the audit's axe injection is not a navigation.
+AfterScenario({ name: 'sdods:perf', tags: '@perf' }, async ({ config, scenario, apiContext, $testInfo }) => {
+    await noting(scenario, () => judgeScenarioPerf({ config, scenario, apiContext, testInfo: $testInfo }));
+});
+// `a11y.scope: every-page` only; both are no-ops under the default `final`.
+BeforeScenario({ name: 'sdods:a11y:watch', tags: `@a11y and ${PAGE}` }, async ({ page, config, scenario }) => {
+    watchA11yPages({ page, config, scenario });
+});
+AfterStep({ name: 'sdods:a11y:page', tags: `@a11y and ${PAGE}` }, async ({ page, config, scenario, $bddContext }) => {
+    await noting(scenario, () => auditAfterStep({
+        page,
+        config,
+        scenario,
+        step: $bddContext.step.title,
+        failed: $bddContext.step.error !== undefined,
+    }));
+});
+AfterScenario({ name: 'sdods:a11y', tags: `@a11y and ${PAGE}` }, async ({ page, config, scenario, $testInfo }) => {
+    await noting(scenario, () => auditScenarioEnd({ page, config, scenario, testInfo: $testInfo }));
+});
+//# sourceMappingURL=hooks.js.map

package/dist/quality/index.d.ts

@@ -0,0 +1,4 @@
+export { collectGateEvidence, evaluateGates, gateReportsFromRunnerJson, gateTotalsFromRunnerStats, hasGates, ingestGateJudge, type GateEvidence, type GateResult, type GateRow, type GateTotals, } from './gates.js';
+export type { A11yPageReport, A11yScenarioReport, A11yScenarioStatus } from './a11y-scenario.js';
+export type { PerfBreach, PerfScenarioReport } from './perf-scenario.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/quality/index.js

@@ -0,0 +1,2 @@
+export { collectGateEvidence, evaluateGates, gateReportsFromRunnerJson, gateTotalsFromRunnerStats, hasGates, ingestGateJudge, } from './gates.js';
+//# sourceMappingURL=index.js.map

package/dist/quality/perf-scenario.d.ts

@@ -0,0 +1,149 @@
+import type { Page } from '@playwright/test';
+import { type ApiSnapshot } from '@sdods/contracts';
+import { type BudgetConfig, type VitalsRecording } from '../perf/vitals.js';
+/**
+ * The budget check an `@perf` tag buys. Before this existed `perf.budgets` was enforced only by
+ * explicit steps, `apiP95Ms` only by a sampling step, and the tag itself by nothing.
+ *
+ * Two halves, because a page has to be watched while the scenario runs and judged after it:
+ *
+ *   · `watchNavigations` (UI and hybrid, before the scenario) installs an init script in the page.
+ *     Every document the main frame loads reports its own vitals through an exposed binding, once
+ *     the load event has finished. The document reports itself rather than being read from Node on
+ *     a `load` event, because a Node-side read races the next navigation: a step that clicks a link
+ *     straight after the load would have its read land in the NEXT document, whose load has not
+ *     ended, and report a slow page that does not exist.
+ *   · `judgeScenarioPerf` (every layer, after the scenario) compares every recorded navigation with
+ *     the page budgets (`pageLoadMs`, `lcpMs`, `fcpMs`, `ttfbMs`), and the p95 of the scenario's live
+ *     API response times (plus any latency samples it took) with `apiP95Ms`.
+ *
+ * The rules are the explicit steps' rules. A vital a budget applies to that did not fire FAILS,
+ * naming why. An `@perf` scenario on an environment with no budgets at all FAILS as CONFIG_INVALID.
+ * A scenario where no configured budget applied to anything it measured FAILS: it proved nothing.
+ * An API call replayed from a HAR is not a measurement of this environment and is not counted.
+ *
+ * One thing is NOT a failure: a document the scenario left before it could report (a navigation
+ * away before the document painted and its LCP window closed). That is not the application being slow; it is
+ * listed under `unreported` in the attachment so the gap is visible.
+ */
+/**
+ * How long a document waits, after its load event, for its first contentful paint before reporting.
+ * A client-rendered page (SauceDemo's login is one) routinely paints AFTER `load`: reading its
+ * vitals at the load event reports FCP and LCP as never having fired on a page that painted a
+ * moment later. A page that has not painted after this long reports what it has, and a budget on
+ * the missing vital then fails naming why.
+ */
+export declare const PAINT_WAIT_MS = 5000;
+/**
+ * How long the judge waits, at scenario end, for documents that loaded but have not reported: the
+ * paint wait plus the LCP settle window, with room for the binding round-trip.
+ */
+export declare const REPORT_WAIT_MS: number;
+/** The binding each document reports through. Namespaced so it cannot collide with the app. */
+export declare const VITALS_BINDING = "__sdodsReportVitals";
+/**
+ * Resolves once the document has painted its first content, or after `paintWaitMs` when it never
+ * does. Runs in the page: it is serialised into the init script below and into `waitForSettle`, so
+ * the `@perf` and `@a11y` hooks agree on when a page has settled.
+ */
+export declare function firstPaintOrTimeout(paintWaitMs: number): Promise<void>;
+/**
+ * The script every document in the page runs. Only the top-level document reports: after its load
+ * event, once it has painted (or `paintWaitMs` has passed), and after the same LCP settle window the
+ * explicit `I record the page vitals` step uses, so both measure a page the same way.
+ */
+export declare function vitalsInitScript(settleMs?: number, paintWaitMs?: number, binding?: string): string;
+/**
+ * Waits, from Node, for the point at which the `@perf` init script reads a document: its load event
+ * (bounded by `LOAD_WAIT_MS`), its first contentful paint (bounded by `PAINT_WAIT_MS`), then the LCP
+ * settle window. Never throws: a page that will not settle is audited as it stands, and a page that
+ * navigates away while this waits is the caller's to notice.
+ */
+export declare function waitForSettle(page: Page, opts?: {
+    paintWaitMs?: number;
+    settleMs?: number;
+    loadWaitMs?: number;
+}): Promise<void>;
+/**
+ * Starts recording the vitals of every document the page's main frame loads. Must run before the
+ * scenario's first navigation, which is why the hook that calls it is a `BeforeScenario`.
+ */
+export declare function watchNavigations(deps: {
+    page: Page;
+    scenario: {
+        data: {
+            browser?: string;
+        };
+    };
+}): Promise<void>;
+export interface PerfBreach {
+    budget: string;
+    limitMs: number;
+    /** `null` when the value was never measured. */
+    measuredMs: number | null;
+    where: string;
+    message: string;
+}
+/** What `sdods/perf-scenario` contains, and what the `perfBudgets` process gate reads back. */
+export interface PerfScenarioReport {
+    kind: 'perf-scenario';
+    runnerProject: string;
+    fingerprint: string;
+    retry: number;
+    env: string;
+    status: 'judged' | 'skipped';
+    reason?: string;
+    budgets: Record<string, number>;
+    navigations: {
+        url: string;
+        vitals: VitalsRecording['vitals'];
+        unmeasured: VitalsRecording['unmeasured'];
+    }[];
+    /** Documents that loaded but were left before they could report. Listed, not failed. */
+    unreported: number;
+    api: {
+        liveCalls: number;
+        replayedFromHar: number;
+        samples: number;
+        p95Ms: number | null;
+    };
+    /** How many budget comparisons were made. Zero means the scenario proved nothing. */
+    judged: number;
+    breaches: PerfBreach[];
+}
+export interface PerfScenarioDeps {
+    config: BudgetConfig;
+    scenario: {
+        data: {
+            runnerProject: string;
+            fingerprint: string;
+            retry: number;
+            layer?: string;
+        };
+        file(rel: string): string;
+    };
+    apiContext: {
+        history: readonly ApiSnapshot[];
+    };
+    testInfo: {
+        status?: string;
+        attach(name: string, opts: {
+            path: string;
+            contentType: string;
+        }): Promise<void>;
+    };
+}
+/** Pure: every comparison and breach for a set of recordings and API timings. */
+export declare function judgeBudgets(input: {
+    budgets: Record<string, number>;
+    recordings: readonly VitalsRecording[];
+    apiMs: readonly number[];
+    apiReplayed: number;
+    envName: string;
+}): {
+    judged: number;
+    breaches: PerfBreach[];
+    p95Ms: number | null;
+};
+export declare function judgeScenarioPerf(deps: PerfScenarioDeps): Promise<PerfScenarioReport>;
+//# sourceMappingURL=perf-scenario.d.ts.map