@lullabot/playwright-testing 0.1.0

package/lib/github/attachments.js

@@ -0,0 +1,297 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AttachmentUploader = void 0;
+exports.mimeTypeFor = mimeTypeFor;
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+/**
+ * Uploads files to GitHub's user-attachments endpoint — the programmatic
+ * equivalent of dragging an image into a comment box, which is the only way to
+ * get an image into a job summary or a PR comment.
+ *
+ * The endpoint is undocumented, and two things about it drive this design:
+ *
+ * 1. It only accepts *user* tokens. The Actions `GITHUB_TOKEN` and GitHub App
+ *    installation tokens are both refused with a 404, so the feature is opt-in
+ *    behind a PAT and every caller must cope with it being switched off.
+ * 2. The URL it returns is a renderer-only handle. Fetching it directly is a
+ *    404 with or without credentials; GitHub rewrites it to a signed, short
+ *    lived URL when it renders the surrounding Markdown. Never try to verify
+ *    an upload by reading it back — a 201 with a URL in the body is the only
+ *    success signal there is.
+ *
+ * What it accepts is no longer guesswork, even though the endpoint has no
+ * documentation of its own: `gh --attach` (CLI 2.99.0) is a supported wrapper
+ * over the same upload, and GitHub documents its formats and size limits. That
+ * flag needs the same kind of user token, which is why it is no help here —
+ * see docs/working-with-tests/screenshots-in-ci.md.
+ *
+ * A single endpoint failure still disables the uploader for the rest of the
+ * run: retrying an endpoint that has answered 404 once only wastes CI time,
+ * and nothing here is worth failing a build over.
+ */
+const UPLOAD_ORIGIN = 'https://uploads.github.com';
+const DEFAULT_MAX_UPLOADS = 20;
+const DEFAULT_MAX_TOTAL_BYTES = 20 * 1024 * 1024;
+/**
+ * GitHub's own ceiling for a single image or GIF, as documented for the
+ * `gh --attach` flag over this endpoint. Video is 10 MB on free plans and
+ * 100 MB on paid ones; this uploader carries screenshots, so the image number
+ * is the one that binds. Checking it here turns a rejection that would stop
+ * the run into one skipped file.
+ */
+const DEFAULT_MAX_FILE_BYTES = 10 * 1024 * 1024;
+const DEFAULT_TIMEOUT_MS = 15000;
+const MIME_TYPES = {
+    '.png': 'image/png',
+    '.jpg': 'image/jpeg',
+    '.jpeg': 'image/jpeg',
+    '.gif': 'image/gif',
+    '.webp': 'image/webp',
+    '.svg': 'image/svg+xml',
+    '.webm': 'video/webm',
+    '.mp4': 'video/mp4',
+    '.mov': 'video/quicktime',
+};
+/**
+ * Guess a MIME type from a file extension. The endpoint requires one, and it
+ * is what decides whether GitHub renders the attachment inline.
+ */
+function mimeTypeFor(filePath) {
+    return MIME_TYPES[path.extname(filePath).toLowerCase()] ?? 'application/octet-stream';
+}
+class AttachmentUploader {
+    token;
+    repositoryId;
+    maxUploads;
+    maxTotalBytes;
+    maxFileBytes;
+    timeoutMs;
+    fetchImpl;
+    log;
+    disabled;
+    reason = null;
+    lastSkip = null;
+    stats = { uploaded: 0, skipped: 0, bytes: 0 };
+    constructor(options = {}) {
+        this.token = String(options.token ?? '');
+        this.repositoryId = String(options.repositoryId ?? '');
+        this.maxUploads = options.maxUploads ?? DEFAULT_MAX_UPLOADS;
+        this.maxTotalBytes = options.maxTotalBytes ?? DEFAULT_MAX_TOTAL_BYTES;
+        this.maxFileBytes = options.maxFileBytes ?? DEFAULT_MAX_FILE_BYTES;
+        this.timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+        this.fetchImpl = options.fetchImpl ?? globalThis.fetch;
+        this.log = options.log ?? (() => { });
+        if (!this.token) {
+            this.disabled = true;
+            this.reason = 'no upload token configured';
+        }
+        else if (!this.repositoryId) {
+            this.disabled = true;
+            this.reason = 'no repository ID available';
+        }
+        else {
+            this.disabled = false;
+        }
+    }
+    get enabled() {
+        return !this.disabled;
+    }
+    /** Why uploads stopped, or null while they are still running. */
+    get disabledReason() {
+        return this.reason;
+    }
+    /**
+     * Why the most recent call returned null, or null when it returned a URL.
+     *
+     * A file that could not be read and one the size limits refused both come
+     * back as null from a still-running uploader, and a caller that cannot tell
+     * them apart reports the wrong problem — an oversized screenshot reads as a
+     * missing one, which sends the reader hunting for a path mapping that is not
+     * wrong.
+     */
+    get lastSkipReason() {
+        return this.lastSkip;
+    }
+    getStats() {
+        return { ...this.stats };
+    }
+    /**
+     * Upload one file and return the URL to embed, or null if the upload did not
+     * happen. Never throws and never rejects: a broken undocumented endpoint
+     * must not fail anyone's build.
+     */
+    async upload(filePath, displayName) {
+        if (this.disabled)
+            return this.skip('disabled');
+        let body;
+        try {
+            body = fs.readFileSync(filePath);
+        }
+        catch {
+            // A missing file is this file's problem, not a reason to stop the run.
+            return this.skip('unreadable', `Attachment upload skipped, unreadable file: ${filePath}`);
+        }
+        return this.uploadBuffer(body, displayName ?? path.basename(filePath), mimeTypeFor(filePath));
+    }
+    /**
+     * Upload bytes that never touched the disk. Playwright's JSON reporter
+     * inlines attachments added with `testInfo.attach({ body })` as base64, which
+     * is how the accessibility screenshots arrive.
+     */
+    async uploadBuffer(body, displayName, contentType) {
+        if (this.disabled)
+            return this.skip('disabled');
+        const size = body.length;
+        const name = sanitizeName(displayName);
+        // A count budget is reached once and stays reached, so stop there.
+        if (this.stats.uploaded >= this.maxUploads) {
+            return this.disable(`upload limit of ${this.maxUploads} reached`);
+        }
+        // Size limits are the individual file's problem. GitHub refuses one over
+        // its own ceiling, and a run has only so many bytes to spend; either way
+        // one large screenshot must not cost the run every smaller one behind it,
+        // so skip it and carry on. `maxUploads` bounds how often this can repeat.
+        if (size > this.maxFileBytes) {
+            return this.skip('too-large', `Attachment skipped, ${size} bytes is over the ${this.maxFileBytes} byte limit: ${name}`);
+        }
+        if (this.stats.bytes + size > this.maxTotalBytes) {
+            const left = this.maxTotalBytes - this.stats.bytes;
+            return this.skip('budget', `Attachment skipped, ${size} bytes does not fit the ${left} bytes left in the budget: ${name}`);
+        }
+        const query = new URLSearchParams({
+            name,
+            content_type: contentType,
+            repository_id: this.repositoryId,
+        });
+        try {
+            const response = await this.fetchImpl(`${UPLOAD_ORIGIN}/user-attachments/assets?${query}`, {
+                method: 'POST',
+                headers: {
+                    Authorization: `Bearer ${this.token}`,
+                    Accept: 'application/json',
+                    // Required. Without it the endpoint answers 400 "Invalid
+                    // Content-Type", and fetch sends no default for a Buffer body.
+                    'Content-Type': contentType,
+                },
+                body,
+                signal: AbortSignal.timeout(this.timeoutMs),
+            });
+            const text = await response.text();
+            if (!response.ok) {
+                return this.disable(`HTTP ${response.status} from the attachments endpoint: ${summarize(text)}`);
+            }
+            const url = parseUploadUrl(text);
+            if (!url) {
+                return this.disable('the attachments endpoint returned no URL');
+            }
+            this.stats.uploaded++;
+            this.stats.bytes += size;
+            this.lastSkip = null;
+            return url;
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            return this.disable(`upload failed: ${message}`);
+        }
+    }
+    /** Switch the uploader off for the rest of the run and say why. */
+    disable(reason) {
+        this.disabled = true;
+        this.reason = reason;
+        this.log(`Attachment uploads disabled — ${reason}.`);
+        return this.skip('disabled');
+    }
+    /**
+     * Record one file that was not uploaded, and why. Most reasons leave the
+     * uploader running; `disable()` routes through here for the one that does
+     * not, so `lastSkipReason` is set however a call came back null.
+     */
+    skip(reason, message) {
+        this.stats.skipped++;
+        this.lastSkip = reason;
+        if (message)
+            this.log(message);
+        return null;
+    }
+}
+exports.AttachmentUploader = AttachmentUploader;
+/**
+ * Pull the asset URL out of a response body. The endpoint has been seen to
+ * answer with `url`; `href` appears in third-party write-ups, so accept both.
+ */
+function parseUploadUrl(text) {
+    try {
+        const parsed = JSON.parse(text);
+        const url = parsed?.url ?? parsed?.href ?? parsed?.asset?.href;
+        return typeof url === 'string' && url ? url : null;
+    }
+    catch {
+        return null;
+    }
+}
+/** Condense a response body into one line worth putting in a CI log. */
+function summarize(text) {
+    const collapsed = text.replace(/\s+/g, ' ').trim();
+    if (!collapsed)
+        return 'no response body';
+    return collapsed.length > 200 ? `${collapsed.slice(0, 200)}…` : collapsed;
+}
+/**
+ * Trim leading and trailing dashes.
+ *
+ * An index scan rather than `/^-+|-+$/`, because that pattern backtracks
+ * quadratically over a long run of dashes and this input is derived from test
+ * titles — which nobody controls.
+ */
+function trimDashes(value) {
+    let start = 0;
+    let end = value.length;
+    while (start < end && value[start] === '-')
+        start++;
+    while (end > start && value[end - 1] === '-')
+        end--;
+    return value.slice(start, end);
+}
+/**
+ * Reduce a file name to something safe to put in a query string. Test titles
+ * reach this by way of snapshot file names and can contain anything.
+ */
+function sanitizeName(name) {
+    // Truncate before trimming, so a trailing dash left by the cut goes too.
+    const cleaned = name.replace(/[^A-Za-z0-9._-]+/g, '-').slice(0, 120);
+    return trimDashes(cleaned) || 'attachment';
+}

package/lib/github/failure-summary.d.ts

@@ -0,0 +1,144 @@
+import { AttachmentUploader, AttachmentUploaderOptions } from './attachments';
+import { PathPrefix, PathResolution } from './report-paths';
+export type ImageKind = 'diff' | 'actual' | 'expected' | 'previous' | 'a11y' | 'screenshot';
+export interface FailureImage {
+    /** Attachment name as it appears in the report. */
+    name: string;
+    /** Path on disk. Attachments the reporter inlined have none. */
+    filePath?: string;
+    /**
+     * Base64 bytes, for attachments added with `testInfo.attach({ body })`. The
+     * JSON reporter inlines those rather than writing them out, which is how the
+     * accessibility screenshots arrive.
+     */
+    body?: string;
+    contentType: string;
+    kind: ImageKind;
+    /** Populated once the image has been uploaded. */
+    url?: string;
+    /**
+     * Set when the recorded path could not be found on this side of a container
+     * boundary. Distinguishes "there was nothing to upload with" from "there was
+     * nowhere to upload to", which otherwise render identically.
+     */
+    unreadable?: boolean;
+    /**
+     * Set when the uploader would not carry the image: over GitHub's 10 MB
+     * ceiling, or over what is left of the run's byte budget. Uploading carried
+     * on around it, so this is one image's story rather than the run's.
+     */
+    oversize?: boolean;
+}
+export interface FailedTest {
+    title: string;
+    file: string;
+    line: number;
+    /** Display status, including the synthetic `flaky` status for a retry that passed. */
+    status: string;
+    /** First error message, trimmed to something a comment can carry. */
+    error?: string;
+    images: FailureImage[];
+}
+export interface FailureReport {
+    tests: FailedTest[];
+    totalFailed: number;
+    totalFlaky: number;
+    totalImages: number;
+}
+/** Which snapshot images to include. `diff` is the one that shows the problem. */
+export type IncludeMode = 'diff' | 'all';
+/**
+ * Prefix of the HTML comment carrying the failure count in a generated comment
+ * body. Invisible when rendered, and greppable by a workflow that has to decide
+ * whether the run is worth commenting on.
+ */
+export declare const FAILURE_MARKER_PREFIX = "<!-- playwright-testing-failures: ";
+/** Machine-readable flaky-test count used by the comment assembly action. */
+export declare const FLAKE_MARKER_PREFIX = "<!-- playwright-testing-flakes: ";
+/**
+ * Neutralise text that the Actions runner would redact.
+ *
+ * Job summaries pass through the runner's secret masking (comments posted over
+ * the API do not). The masker replaces `Bearer <value>` with `***` and takes
+ * the following characters with it, so a test name or error containing the
+ * word can silently swallow the Markdown that follows it. A zero-width space
+ * inside the word reads identically and no longer matches.
+ */
+export declare function defuseMaskTriggers(text: string): string;
+/**
+ * Parse a Playwright JSON report into the failures worth reporting.
+ */
+export declare function parseFailures(reportPath: string, include?: IncludeMode): FailureReport;
+/** What re-rooting the report's attachment paths achieved. */
+export interface PathResolutionSummary {
+    /** Attachments whose recorded path had to be rewritten to be readable. */
+    remapped: number;
+    /** The mappings that were used, for the log line. */
+    used: PathPrefix[];
+    /** Recorded paths with no readable file behind them, on either side. */
+    unreadable: string[];
+}
+/**
+ * Point every image at a file this process can open, and note the ones where
+ * that was not possible.
+ *
+ * Runs whether or not uploads are switched on: an unreachable attachment is
+ * worth reporting even on a fork build that was never going to upload it,
+ * because it is the same misconfiguration either way.
+ */
+export declare function resolveImagePaths(report: FailureReport, resolve: (filePath: string) => PathResolution): PathResolutionSummary;
+/**
+ * Upload every collected image, annotating each one with its URL. Images the
+ * uploader declines are simply left without a URL, which the renderers treat
+ * as "link to the artifact instead".
+ */
+export declare function uploadImages(report: FailureReport, uploader: AttachmentUploader): Promise<void>;
+export interface SummaryOptions {
+    artifactHint?: string;
+    /**
+     * Why uploading did not happen at all, phrased to sit inside a sentence —
+     * `no upload token configured`, say. Leave unset when uploads ran.
+     */
+    uploadReason?: string;
+}
+/**
+ * Render the job summary: a heading, a headline, and one collapsed block per
+ * test holding its images.
+ *
+ * Deliberately table-free. Test titles and error messages are arbitrary text,
+ * and the runner's masking can eat a cell delimiter and corrupt a whole row.
+ */
+export declare function generateSummary(report: FailureReport, options?: SummaryOptions): string;
+/**
+ * Render the pull request comment. This is where the screenshots go.
+ *
+ * A comment is rendered afresh every time it is read, so an attachment resolves
+ * however recently it was uploaded. A job summary is rendered once when the job
+ * ends, before a just-uploaded attachment is resolvable, and that result is
+ * what everyone sees from then on.
+ *
+ * The cost is emailed notifications: those are rendered once when sent, and the
+ * signed URLs behind these images expire minutes later, so the images will be
+ * broken in the email even though they are fine on the web.
+ */
+export interface FailureCommentOptions {
+    summaryUrl?: string;
+    title?: string;
+    uploadReason?: string;
+    /** Adapter namespace for the machine-readable failure marker. */
+    failureMarkerPrefix?: string;
+    /** Adapter namespace for the machine-readable flaky-test marker. */
+    flakeMarkerPrefix?: string;
+}
+export declare function generateComment(report: FailureReport, options?: FailureCommentOptions): string;
+export declare function uploaderFromEnvironment(overrides?: AttachmentUploaderOptions): AttachmentUploader;
+/**
+ * CLI entry point. Runs once and writes both outputs, so images are never
+ * uploaded twice for the same run.
+ */
+export interface FailureSummaryMainOptions {
+    commandName?: string;
+    failureMarkerPrefix?: string;
+    flakeMarkerPrefix?: string;
+}
+export declare function main(args?: string[], adapter?: FailureSummaryMainOptions): Promise<void>;