@lullabot/playwright-testing 0.1.0

package/lib/github/a11y-summary.d.ts

@@ -0,0 +1,55 @@
+/** A test that has accessibility-related annotations and/or attachments. */
+export interface A11yTestResult {
+    /** Test title path, e.g. "a11y > homepage has accessibility violations" */
+    title: string;
+    /** Source file, e.g. "tests/a11y.spec.ts" */
+    file: string;
+    /** Line number in the source file. */
+    line: number;
+    /** Accessibility annotations from testInfo.annotations. */
+    annotations: Array<{
+        type: string;
+        description?: string;
+    }>;
+    /** Parsed WCAG scan violations (from a11y-wcag-scan-results attachment). */
+    violations: NormalizedViolation[];
+}
+interface NormalizedViolation {
+    rule: string;
+    impact: string;
+    description: string;
+    helpUrl: string;
+    targets: string[];
+}
+/** Parsed report with only accessibility-relevant data. */
+export interface A11yReport {
+    tests: A11yTestResult[];
+    totalViolations: number;
+    totalBaselined: number;
+    totalStale: number;
+}
+/**
+ * Parse a Playwright JSON report and extract accessibility-related test data.
+ */
+export declare function parseA11yResults(reportPath: string): A11yReport;
+/**
+ * Generate Markdown for $GITHUB_STEP_SUMMARY.
+ */
+export declare function generateSummary(report: A11yReport): string;
+/**
+ * Generate GitHub Actions annotation commands (::error, ::warning).
+ *
+ * All interpolated values are escaped — CSS selectors from axe-core targets
+ * are extracted from page DOM and could contain newlines or `%` sequences
+ * that would otherwise let a crafted page forge additional workflow commands.
+ */
+export declare function generateAnnotations(report: A11yReport): string;
+/**
+ * CLI entry point. Called automatically when this file is run directly
+ * (e.g. `node lib/github/a11y-summary.js`), or can be imported and
+ * called from the bin wrapper.
+ */
+export declare function main(args?: string[], adapter?: {
+    commandName?: string;
+}): void;
+export {};

package/lib/github/a11y-summary.js

@@ -0,0 +1,383 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseA11yResults = parseA11yResults;
+exports.generateSummary = generateSummary;
+exports.generateAnnotations = generateAnnotations;
+exports.main = main;
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+/** Annotation types that identify accessibility-relevant test data. */
+const A11Y_ANNOTATION_TYPES = new Set([
+    '@a11y',
+    'Accessibility',
+    'Baselined a11y violation',
+    'Stale a11y baseline entry',
+]);
+/**
+ * Escape a string for use inside a GitHub Actions workflow-command message
+ * body (the text after `::`). Without this, a crafted value containing a
+ * newline or `%` sequence could forge additional workflow commands.
+ * See https://docs.github.com/en/actions/reference/workflow-commands-for-github-actions
+ */
+function escapeWorkflowCommand(value) {
+    return String(value)
+        .replace(/%/g, '%25')
+        .replace(/\r/g, '%0D')
+        .replace(/\n/g, '%0A');
+}
+/**
+ * Escape a string for use inside a GitHub Actions workflow-command property
+ * value (e.g. `file=...`, `title=...`). Property values additionally require
+ * escaping of `:` and `,` on top of the message-body escapes.
+ */
+function escapeWorkflowCommandProperty(value) {
+    return escapeWorkflowCommand(value)
+        .replace(/:/g, '%3A')
+        .replace(/,/g, '%2C');
+}
+/**
+ * Escape a string for use inside a Markdown table cell. Backslashes must be
+ * escaped first (they are Markdown's escape character), then pipes and
+ * newlines can be substituted safely.
+ */
+function escapeMarkdownTableCell(value) {
+    return String(value)
+        .replace(/\\/g, '\\\\')
+        .replace(/\|/g, '\\|')
+        .replace(/\r?\n/g, ' ');
+}
+/**
+ * Escape a string for use inside a Markdown inline code span (backticks).
+ * A literal backtick would close the span early and let the remainder render
+ * as Markdown — wrap the value in double backticks and pad literal backticks
+ * with spaces, which is the standard CommonMark escape for code spans.
+ */
+function escapeMarkdownCodeSpan(value) {
+    const s = String(value).replace(/\r?\n/g, ' ');
+    if (!s.includes('`'))
+        return `\`${s}\``;
+    return `\`\` ${s.replace(/``/g, '` `')} \`\``;
+}
+/**
+ * Parse a Playwright JSON report and extract accessibility-related test data.
+ */
+function parseA11yResults(reportPath) {
+    if (!fs.existsSync(reportPath)) {
+        throw new Error(`Playwright JSON report not found at ${reportPath}. ` +
+            'Ensure the Playwright JSON reporter is enabled.');
+    }
+    const report = JSON.parse(fs.readFileSync(reportPath, 'utf8'));
+    const tests = [];
+    let totalViolations = 0;
+    let totalBaselined = 0;
+    let totalStale = 0;
+    walkSuites(report.suites ?? [], '', tests);
+    for (const test of tests) {
+        for (const ann of test.annotations) {
+            if (ann.type === 'Baselined a11y violation')
+                totalBaselined++;
+            if (ann.type === 'Stale a11y baseline entry')
+                totalStale++;
+        }
+        totalViolations += test.violations.length;
+    }
+    return { tests, totalViolations, totalBaselined, totalStale };
+}
+/**
+ * Collect annotations from both test and result levels. Playwright stores
+ * annotations declared at test definition time on `test.annotations`, and
+ * annotations added during execution (e.g. `testInfo.annotations.push()`)
+ * on `result.annotations`. checkAccessibility() uses the latter.
+ */
+function collectAnnotations(test, lastResult) {
+    return [
+        ...(test.annotations ?? []),
+        ...(lastResult?.annotations ?? []),
+    ];
+}
+/** Read and parse the WCAG scan attachment, if present. */
+function parseWcagViolations(attachment) {
+    if (!attachment)
+        return [];
+    try {
+        const raw = attachment.body
+            ? Buffer.from(attachment.body, 'base64').toString('utf8')
+            : attachment.path
+                ? fs.readFileSync(attachment.path, 'utf8')
+                : null;
+        if (!raw)
+            return [];
+        const json = JSON.parse(raw);
+        if (!json?.violations)
+            return [];
+        return json.violations.map((v) => ({
+            rule: v.id,
+            impact: v.impact ?? 'unknown',
+            description: v.description,
+            helpUrl: v.helpUrl,
+            targets: v.nodes
+                ?.flatMap((n) => n.target)
+                .filter((t) => typeof t === 'string') ?? [],
+        }));
+    }
+    catch {
+        // Parse errors are non-fatal — we'll just report without violation details.
+        return [];
+    }
+}
+/** Extract an A11yTestResult from a Playwright test, or null if not a11y-related. */
+function extractTestResult(spec, test, file) {
+    const lastResult = test.results?.[test.results.length - 1];
+    const attachments = lastResult?.attachments ?? [];
+    const a11yAnnotations = collectAnnotations(test, lastResult)
+        .filter((a) => A11Y_ANNOTATION_TYPES.has(a.type));
+    if (a11yAnnotations.length === 0)
+        return null;
+    const violations = parseWcagViolations(attachments.find((a) => a.name === 'a11y-wcag-scan-results'));
+    return {
+        title: spec.title,
+        file,
+        line: spec.line ?? 1,
+        annotations: a11yAnnotations,
+        violations,
+    };
+}
+function walkSuites(suites, parentFile, out) {
+    for (const suite of suites) {
+        const file = suite.file || parentFile;
+        for (const spec of suite.specs ?? []) {
+            for (const test of spec.tests ?? []) {
+                const result = extractTestResult(spec, test, file);
+                if (result)
+                    out.push(result);
+            }
+        }
+        if (suite.suites) {
+            walkSuites(suite.suites, file, out);
+        }
+    }
+}
+/** Headline with pass/fail counts. */
+function renderHeadline(report) {
+    const hasViolations = report.totalViolations > 0;
+    const hasBaselined = report.totalBaselined > 0;
+    const hasStale = report.totalStale > 0;
+    if (!hasViolations && !hasBaselined && !hasStale) {
+        return ':white_check_mark: All accessibility checks passed.\n';
+    }
+    const parts = [];
+    if (hasViolations)
+        parts.push(`**${report.totalViolations}** violation(s)`);
+    if (hasBaselined)
+        parts.push(`**${report.totalBaselined}** baselined`);
+    if (hasStale)
+        parts.push(`**${report.totalStale}** stale baseline entries`);
+    return parts.join(' · ') + '\n';
+}
+/** Icon for an `Accessibility` annotation based on its violation count. */
+function iconForAccessibilityAnnotation(description) {
+    // Descriptions look like "WCAG scan: 0 violations ..." or "WCAG scan: 3 violations ...".
+    const colonIdx = description.lastIndexOf(': ');
+    const afterColon = colonIdx >= 0 ? description.substring(colonIdx + 2) : description;
+    const violationCount = parseInt(afterColon, 10);
+    return violationCount > 0 ? ':x:' : ':white_check_mark:';
+}
+function iconForAnnotation(ann) {
+    switch (ann.type) {
+        case 'Baselined a11y violation':
+            return ':white_check_mark:';
+        case 'Stale a11y baseline entry':
+            return ':warning:';
+        case 'Accessibility':
+            return iconForAccessibilityAnnotation(ann.description ?? '');
+        default:
+            return ':information_source:';
+    }
+}
+/** Render annotation lines, deduplicated by type+description. */
+function renderAnnotations(annotations) {
+    const lines = [];
+    const seen = new Set();
+    for (const ann of annotations) {
+        const key = `${ann.type}:${ann.description ?? ''}`;
+        if (seen.has(key))
+            continue;
+        seen.add(key);
+        lines.push(`${iconForAnnotation(ann)} **${ann.type}**: ${ann.description ?? ''}\n`);
+    }
+    return lines;
+}
+/** Render the violation table for a single test. */
+function renderViolationTable(violations) {
+    if (violations.length === 0)
+        return [];
+    const lines = [
+        '| Rule | Impact | Description | Targets |',
+        '|------|--------|-------------|---------|',
+    ];
+    for (const v of violations) {
+        const shown = v.targets.slice(0, 3).map(escapeMarkdownCodeSpan).join(', ');
+        const targets = v.targets.length > 3
+            ? `${shown}, +${v.targets.length - 3} more`
+            : shown;
+        const rule = escapeMarkdownTableCell(v.rule);
+        const impact = escapeMarkdownTableCell(v.impact);
+        const description = escapeMarkdownTableCell(v.description);
+        lines.push(`| [${rule}](${v.helpUrl}) | ${impact} | ${description} | ${targets} |`);
+    }
+    lines.push('');
+    return lines;
+}
+/**
+ * Generate Markdown for $GITHUB_STEP_SUMMARY.
+ */
+function generateSummary(report) {
+    if (report.tests.length === 0) {
+        return '## Accessibility Results\n\nNo accessibility tests were found in the report.\n';
+    }
+    const lines = ['## Accessibility Results\n', renderHeadline(report)];
+    const seenTitles = new Set();
+    for (const test of report.tests) {
+        const testAnnotations = test.annotations.filter(a => a.type !== '@a11y');
+        const hasTestViolations = test.violations.length > 0;
+        if (testAnnotations.length === 0 && !hasTestViolations)
+            continue;
+        // Skip duplicate test titles (different browsers report the same spec).
+        if (seenTitles.has(test.title))
+            continue;
+        seenTitles.add(test.title);
+        const testLines = [`### ${test.title}\n`];
+        testLines.push(...renderAnnotations(testAnnotations));
+        testLines.push(...renderViolationTable(test.violations));
+        lines.push(...testLines);
+    }
+    return lines.join('\n');
+}
+/**
+ * Generate GitHub Actions annotation commands (::error, ::warning).
+ *
+ * All interpolated values are escaped — CSS selectors from axe-core targets
+ * are extracted from page DOM and could contain newlines or `%` sequences
+ * that would otherwise let a crafted page forge additional workflow commands.
+ */
+function generateAnnotations(report) {
+    const lines = [];
+    for (const test of report.tests) {
+        const file = escapeWorkflowCommandProperty(test.file);
+        const line = escapeWorkflowCommandProperty(String(test.line));
+        // Emit ::error for each violation.
+        for (const v of test.violations) {
+            const targets = v.targets.slice(0, 3).join(', ');
+            const title = escapeWorkflowCommandProperty(`${v.rule} (${v.impact})`);
+            const msg = escapeWorkflowCommand(`a11y: ${v.rule} (${v.impact}) — ${v.description}. Targets: ${targets}`);
+            lines.push(`::error file=${file},line=${line},title=${title}::${msg}`);
+        }
+        // Emit ::warning for stale baseline entries.
+        for (const ann of test.annotations) {
+            if (ann.type === 'Stale a11y baseline entry') {
+                const title = escapeWorkflowCommandProperty('Stale a11y baseline');
+                const msg = escapeWorkflowCommand(ann.description ?? '');
+                lines.push(`::warning file=${file},line=${line},title=${title}::${msg}`);
+            }
+        }
+    }
+    return lines.join('\n');
+}
+/**
+ * CLI entry point. Called automatically when this file is run directly
+ * (e.g. `node lib/github/a11y-summary.js`), or can be imported and
+ * called from the bin wrapper.
+ */
+function main(args = process.argv.slice(2), adapter = {}) {
+    if (args.includes('--help') || args.includes('-h')) {
+        process.stdout.write(`Usage: ${adapter.commandName ?? 'playwright-testing-a11y-summary'} [options]\n\n` +
+            'Options:\n' +
+            '  --report-path=PATH       Playwright JSON report (default: test-results/results.json)\n' +
+            '  --mode=summary           Write a Markdown accessibility summary (default)\n' +
+            '  --mode=annotations       Write GitHub Actions annotations\n' +
+            '  -h, --help               Show this help\n');
+        return;
+    }
+    let mode = 'summary';
+    let reportPath = 'test-results/results.json';
+    for (const arg of args) {
+        if (arg.startsWith('--mode='))
+            mode = arg.slice('--mode='.length);
+        if (arg.startsWith('--report-path='))
+            reportPath = arg.slice('--report-path='.length);
+    }
+    const validModes = new Set(['summary', 'annotations']);
+    if (!validModes.has(mode)) {
+        console.error(`Unknown mode: ${mode}. Use summary or annotations.`);
+        process.exit(2);
+    }
+    // Resolve relative to cwd.
+    const resolvedPath = path.resolve(reportPath);
+    // Missing reports are intentionally non-fatal — the JSON report may be
+    // absent if no tests ran. Any other failure (parse error, permissions, etc.)
+    // should propagate so CI surfaces the real problem instead of silently
+    // reporting zero violations.
+    if (!fs.existsSync(resolvedPath)) {
+        console.error(`Playwright JSON report not found at ${resolvedPath} — skipping a11y summary.`);
+        return;
+    }
+    const report = parseA11yResults(resolvedPath);
+    switch (mode) {
+        case 'summary': {
+            const md = generateSummary(report);
+            const summaryFile = process.env.GITHUB_STEP_SUMMARY;
+            if (summaryFile) {
+                fs.appendFileSync(summaryFile, md);
+                console.log('Accessibility summary written to $GITHUB_STEP_SUMMARY');
+            }
+            else {
+                // Not in GitHub Actions — just print.
+                process.stdout.write(md);
+            }
+            break;
+        }
+        case 'annotations': {
+            const output = generateAnnotations(report);
+            if (output)
+                process.stdout.write(output + '\n');
+            break;
+        }
+    }
+}
+// Auto-invoke when run directly (node lib/github/a11y-summary.js).
+if (require.main === module) {
+    main();
+}

package/lib/github/attachments.d.ts

@@ -0,0 +1,98 @@
+/** The subset of `fetch` this module uses, so tests can substitute their own. */
+export type FetchLike = (url: string, init: {
+    method: string;
+    headers: Record<string, string>;
+    body: Buffer;
+    signal?: AbortSignal;
+}) => Promise<{
+    ok: boolean;
+    status: number;
+    text(): Promise<string>;
+}>;
+export interface AttachmentUploaderOptions {
+    /**
+     * A user PAT. Installation tokens (including the Actions GITHUB_TOKEN) are
+     * rejected by the endpoint, so leaving this unset is the normal case.
+     */
+    token?: string;
+    /** Numeric repository ID — `GITHUB_REPOSITORY_ID` in Actions, not `owner/repo`. */
+    repositoryId?: string | number;
+    /** Stop after this many uploads in one run. Defaults to 20. */
+    maxUploads?: number;
+    /** Stop once this many bytes have been uploaded. Defaults to 20 MiB. */
+    maxTotalBytes?: number;
+    /**
+     * Skip any single file larger than this. Defaults to 10 MiB, which is what
+     * GitHub accepts for an image.
+     */
+    maxFileBytes?: number;
+    /** Per-request timeout. Defaults to 15 seconds. */
+    timeoutMs?: number;
+    fetchImpl?: FetchLike;
+    log?: (message: string) => void;
+}
+/**
+ * Why a call returned null. Every one of these is a null from a method that
+ * never throws, and they need different things said about them: a file that
+ * could not be read is a path problem, one over the size limit is not.
+ */
+export type SkipReason = 'disabled' | 'unreadable' | 'too-large' | 'budget';
+export interface UploadStats {
+    uploaded: number;
+    skipped: number;
+    bytes: number;
+}
+/**
+ * Guess a MIME type from a file extension. The endpoint requires one, and it
+ * is what decides whether GitHub renders the attachment inline.
+ */
+export declare function mimeTypeFor(filePath: string): string;
+export declare class AttachmentUploader {
+    private readonly token;
+    private readonly repositoryId;
+    private readonly maxUploads;
+    private readonly maxTotalBytes;
+    private readonly maxFileBytes;
+    private readonly timeoutMs;
+    private readonly fetchImpl;
+    private readonly log;
+    private disabled;
+    private reason;
+    private lastSkip;
+    private stats;
+    constructor(options?: AttachmentUploaderOptions);
+    get enabled(): boolean;
+    /** Why uploads stopped, or null while they are still running. */
+    get disabledReason(): string | null;
+    /**
+     * Why the most recent call returned null, or null when it returned a URL.
+     *
+     * A file that could not be read and one the size limits refused both come
+     * back as null from a still-running uploader, and a caller that cannot tell
+     * them apart reports the wrong problem — an oversized screenshot reads as a
+     * missing one, which sends the reader hunting for a path mapping that is not
+     * wrong.
+     */
+    get lastSkipReason(): SkipReason | null;
+    getStats(): UploadStats;
+    /**
+     * Upload one file and return the URL to embed, or null if the upload did not
+     * happen. Never throws and never rejects: a broken undocumented endpoint
+     * must not fail anyone's build.
+     */
+    upload(filePath: string, displayName?: string): Promise<string | null>;
+    /**
+     * Upload bytes that never touched the disk. Playwright's JSON reporter
+     * inlines attachments added with `testInfo.attach({ body })` as base64, which
+     * is how the accessibility screenshots arrive.
+     */
+    uploadBuffer(body: Buffer, displayName: string, contentType: string): Promise<string | null>;
+    /** Switch the uploader off for the rest of the run and say why. */
+    private disable;
+    /**
+     * Record one file that was not uploaded, and why. Most reasons leave the
+     * uploader running; `disable()` routes through here for the one that does
+     * not, so `lastSkipReason` is set however a call came back null.
+     */
+    private skip;
+}