@element-hq/element-web-playwright-common 2.4.0 → 3.1.0

package/src/flaky-reporter.ts

@@ -0,0 +1,188 @@
+/*
+Copyright 2024 New Vector Ltd.
+Copyright 2024 The Matrix.org Foundation C.I.C.
+
+SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE files in the repository root for full details.
+*/
+
+/**
+ * Flaky test reporter, creating & updating GitHub issues
+ * Only intended to run from within GitHub Actions
+ */
+
+import type { Reporter, TestCase } from "@playwright/test/reporter";
+
+const REPO = "element-hq/element-web";
+const LABEL = "Z-Flaky-Test";
+const ISSUE_TITLE_PREFIX = "Flaky playwright test: ";
+
+type PaginationLinks = {
+    prev?: string;
+    next?: string;
+    last?: string;
+    first?: string;
+};
+
+const ANSI_COLOUR_REGEX = /[\u001b\u009b][[()#;?]*(?:[0-9]{1,4}(?:;[0-9]{0,4})*)?[0-9A-ORZcf-nqry=><]/g;
+
+// We see quite a few test flakes which are caused by the app exploding
+// so we have some magic strings we check the logs for to better track the flake with its cause
+const SPECIAL_CASES: Record<string, string> = {
+    "ChunkLoadError": "ChunkLoadError",
+    "Unreachable code should not be executed": "Rust crypto panic",
+    "Out of bounds memory access": "Rust crypto memory error",
+};
+
+class FlakyReporter implements Reporter {
+    private flakes = new Map<string, TestCase[]>();
+
+    public onTestEnd(test: TestCase): void {
+        // Ignores flakes on Dendrite and Pinecone as they have their own flakes we do not track
+        if (["Dendrite", "Pinecone"].includes(test.parent.project()!.name!)) return;
+
+        if (test.outcome() === "flaky") {
+            const failures: string[] = [];
+
+            const timedOutRuns = test.results.filter((result) => result.status === "timedOut");
+            const pageLogs = timedOutRuns.flatMap((result) =>
+                result.attachments.filter((attachment) => attachment.name.startsWith("page-")),
+            );
+
+            // If a test failed due to a systemic fault then the test is not flaky, the app is, record it as such.
+            const specialCases = Object.keys(SPECIAL_CASES).filter((log) =>
+                pageLogs.some((attachment) => attachment.name.startsWith("page-") && attachment.body?.includes(log)),
+            );
+            if (specialCases.length > 0) {
+                failures.push(...specialCases.map((specialCase) => SPECIAL_CASES[specialCase]));
+            }
+
+            // Check for fixtures failing to set up
+            const errorMessages = timedOutRuns
+                .map((r) => r.error?.message?.replace(ANSI_COLOUR_REGEX, ""))
+                .filter(Boolean) as string[];
+            for (const error of errorMessages) {
+                if (error.startsWith("Fixture") && error.endsWith("exceeded during setup.")) {
+                    failures.push(error);
+                }
+            }
+
+            if (failures.length < 1) {
+                failures.push(`${test.location.file.split("playwright/e2e/")[1]}: ${test.title}`);
+            }
+
+            for (const title of failures) {
+                if (!this.flakes.has(title)) {
+                    this.flakes.set(title, []);
+                }
+                this.flakes.get(title)!.push(test);
+            }
+        }
+    }
+
+    /**
+     * Parse link header to retrieve pagination links
+     * @see https://docs.github.com/en/rest/using-the-rest-api/using-pagination-in-the-rest-api?apiVersion=2022-11-28#using-link-headers
+     * @param link link header from response or undefined
+     * @returns an empty object if link is undefined otherwise returns a map from type to link
+     */
+    private parseLinkHeader(link: string): PaginationLinks {
+        /**
+         * link looks like:
+         * <https://api.github.com/repositories/1300192/issues?page=2>; rel="prev", <https://api.github.com/repositories/1300192/issues?page=4>;
+         */
+        const map: PaginationLinks = {};
+        if (!link) return map;
+        const matches = link.matchAll(/(<(?<link>.+?)>; rel="(?<type>.+?)")/g);
+        for (const match of matches) {
+            const { link, type } = match.groups!;
+            map[type as keyof PaginationLinks] = link;
+        }
+        return map;
+    }
+
+    /**
+     * Fetch all flaky test issues that were updated since Jan-1-2024
+     * @returns A promise that resolves to a list of issues
+     */
+    async getAllIssues(): Promise<any[]> {
+        const issues = [];
+        const { GITHUB_TOKEN, GITHUB_API_URL } = process.env;
+        // See https://docs.github.com/en/rest/issues/issues?apiVersion=2022-11-28#list-repository-issues
+        let url = `${GITHUB_API_URL}/repos/${REPO}/issues?labels=${LABEL}&state=all&per_page=100&sort=updated&since=2024-01-01`;
+        const headers = {
+            Authorization: `Bearer ${GITHUB_TOKEN}`,
+            Accept: "application / vnd.github + json",
+        };
+        while (url) {
+            // Fetch issues and add to list
+            const issuesResponse = await fetch(url, { headers });
+            const fetchedIssues = await issuesResponse.json();
+            issues.push(...fetchedIssues);
+
+            // Get the next link for fetching more results
+            const linkHeader = issuesResponse.headers.get("Link")!;
+            const parsed = this.parseLinkHeader(linkHeader);
+            url = parsed.next!;
+        }
+        return issues;
+    }
+
+    public async onExit(): Promise<void> {
+        if (this.flakes.size === 0) {
+            console.log("No flakes found");
+            return;
+        }
+
+        console.log("Found flakes: ");
+        for (const flake of this.flakes) {
+            console.log(flake);
+        }
+
+        const { GITHUB_TOKEN, GITHUB_API_URL, GITHUB_SERVER_URL, GITHUB_REPOSITORY, GITHUB_RUN_ID } = process.env;
+        if (!GITHUB_TOKEN) return;
+
+        const issues = await this.getAllIssues();
+        for (const [flake, results] of this.flakes) {
+            const title = ISSUE_TITLE_PREFIX + "`" + flake + "`";
+            const existingIssue = issues.find((issue) => issue.title === title);
+            const headers = { Authorization: `Bearer ${GITHUB_TOKEN}` };
+            const body = `${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}`;
+
+            const labels = [LABEL, ...results.map((test) => `${LABEL}-${test.parent.project()?.name}`)];
+
+            if (existingIssue) {
+                console.log(`Found issue ${existingIssue.number} for ${flake}, adding comment...`);
+                // Ensure that the test is open
+                await fetch(existingIssue.url, {
+                    method: "PATCH",
+                    headers,
+                    body: JSON.stringify({ state: "open" }),
+                });
+                await fetch(`${existingIssue.url}/labels`, {
+                    method: "POST",
+                    headers,
+                    body: JSON.stringify({ labels }),
+                });
+                await fetch(`${existingIssue.url}/comments`, {
+                    method: "POST",
+                    headers,
+                    body: JSON.stringify({ body }),
+                });
+            } else {
+                console.log(`Creating new issue for ${flake}...`);
+                await fetch(`${GITHUB_API_URL}/repos/${REPO}/issues`, {
+                    method: "POST",
+                    headers,
+                    body: JSON.stringify({
+                        title,
+                        body,
+                        labels: [...labels],
+                    }),
+                });
+            }
+        }
+    }
+}
+
+export default FlakyReporter;

package/src/stale-screenshot-reporter.ts

@@ -56,11 +56,7 @@ class StaleScreenshotReporter implements Reporter {
         this.success = false;
     }
 
-    public async onExit(): Promise<void> {
-        if (this.failing.size) {
-            console.error(`${this.failing.size} tests failed, skipping stale screenshot reporter.`);
-        }
-
+    private async checkStaleScreenshots(): Promise<void> {
         if (!this.snapshotRoots.size) {
             this.error("No snapshot directories found, did you set the snapshotDir in your Playwright config?", "");
             return;
@@ -90,6 +86,14 @@ class StaleScreenshotReporter implements Reporter {
                 this.error("Stale screenshot file", screenshot);
             }
         }
+    }
+
+    public async onExit(): Promise<void> {
+        if (this.failing.size) {
+            this.error(`${this.failing.size} tests failed, skipping stale screenshot reporter.`, "");
+        } else {
+            await this.checkStaleScreenshots();
+        }
 
         if (!this.success) {
             process.exit(1);

package/src/testcontainers/index.ts

@@ -6,11 +6,13 @@ Please see LICENSE files in the repository root for full details.
 */
 
 export { PostgreSqlContainer, StartedPostgreSqlContainer } from "@testcontainers/postgresql";
+export { makePostgres } from "./postgres.js";
 export type { HomeserverInstance, HomeserverContainer, StartedHomeserverContainer } from "./HomeserverContainer.js";
 export { type SynapseConfig, SynapseContainer, StartedSynapseContainer } from "./synapse.js";
 export {
     type MasConfig,
     MatrixAuthenticationServiceContainer,
     StartedMatrixAuthenticationServiceContainer,
+    makeMas,
 } from "./mas.js";
 export { type MailpitClient, MailpitContainer, StartedMailpitContainer } from "./mailpit.js";

package/src/testcontainers/mas.ts

@@ -11,6 +11,7 @@ import {
     type StartedTestContainer,
     Wait,
     type ExecResult,
+    type StartedNetwork,
 } from "testcontainers";
 import { type StartedPostgreSqlContainer } from "@testcontainers/postgresql";
 import * as YAML from "yaml";
@@ -23,6 +24,7 @@ import { type Credentials } from "../utils/api.js";
 //   curl -sL https://element-hq.github.io/matrix-authentication-service/config.schema.json \
 //     | npx json-schema-to-typescript -o packages/element-web-playwright-common/src/testconainers/mas-config.ts
 import type { RootConfig as MasConfig } from "./mas-config.js";
+import type { Logger } from "../utils/logger.js";
 
 export { type MasConfig };
 
@@ -156,6 +158,7 @@ export class MatrixAuthenticationServiceContainer extends GenericContainer {
         super(image);
 
         const initialConfig = deepCopy(DEFAULT_CONFIG);
+        initialConfig.database.host = db.getHostname();
         initialConfig.database.username = db.getUsername();
         initialConfig.database.password = db.getPassword();
 
@@ -205,6 +208,7 @@ export class MatrixAuthenticationServiceContainer extends GenericContainer {
             await super.start(),
             `http://localhost:${port}`,
             this.args,
+            this.config.matrix.secret,
         );
     }
 }
@@ -219,6 +223,7 @@ export class StartedMatrixAuthenticationServiceContainer extends AbstractStarted
         container: StartedTestContainer,
         public readonly baseUrl: string,
         private readonly args: string[],
+        public readonly sharedSecret: string,
     ) {
         super(container);
     }
@@ -346,3 +351,19 @@ export class StartedMatrixAuthenticationServiceContainer extends AbstractStarted
         await this.manage("add-email", username, address);
     }
 }
+
+export async function makeMas(
+    postgres: StartedPostgreSqlContainer,
+    network: StartedNetwork,
+    logger: Logger,
+    config: Partial<MasConfig>,
+    name = "mas",
+): Promise<StartedMatrixAuthenticationServiceContainer> {
+    const container = await new MatrixAuthenticationServiceContainer(postgres)
+        .withNetwork(network)
+        .withNetworkAliases(name)
+        .withLogConsumer(logger.getConsumer(name))
+        .withConfig(config)
+        .start();
+    return container;
+}

package/src/testcontainers/postgres.ts

@@ -0,0 +1,40 @@
+/*
+Copyright 2026 Element Creations Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE files in the repository root for full details.
+*/
+
+import { PostgreSqlContainer, type StartedPostgreSqlContainer } from "@testcontainers/postgresql";
+import { type StartedNetwork } from "testcontainers";
+
+import { type Logger } from "../utils/logger.js";
+
+export async function makePostgres(
+    network: StartedNetwork,
+    logger: Logger,
+    name = "postgres",
+): Promise<StartedPostgreSqlContainer> {
+    const container = await new PostgreSqlContainer("postgres:13.3-alpine")
+        .withNetwork(network)
+        .withNetworkAliases(name)
+        .withLogConsumer(logger.getConsumer(name))
+        .withTmpFs({
+            "/dev/shm/pgdata/data": "",
+        })
+        .withEnvironment({
+            PG_DATA: "/dev/shm/pgdata/data",
+        })
+        .withCommand([
+            "-c",
+            "shared_buffers=128MB",
+            "-c",
+            `fsync=off`,
+            "-c",
+            `synchronous_commit=off`,
+            "-c",
+            "full_page_writes=off",
+        ])
+        .start();
+    return container;
+}

package/src/testcontainers/synapse.ts

@@ -184,6 +184,14 @@ const DEFAULT_CONFIG = {
     },
     room_list_publication_rules: [{ action: "allow" }],
     modules: [] as Array<{ module: string; config?: Record<string, unknown> }>,
+    matrix_authentication_service: undefined as
+        | undefined
+        | {
+              enabled?: boolean;
+              endpoint?: string;
+              secret?: string | null;
+              secret_path?: string | null;
+          },
 };
 
 /**
@@ -278,7 +286,22 @@ export class SynapseContainer extends GenericContainer implements HomeserverCont
     }
 
     public withMatrixAuthenticationService(mas?: StartedMatrixAuthenticationServiceContainer): this {
-        this.mas = mas;
+        if (mas) {
+            this.mas = mas;
+            this.withConfig({
+                matrix_authentication_service: {
+                    enabled: true,
+                    endpoint: `http://${mas.getHostname()}:8080/`,
+                    secret: mas.sharedSecret,
+                },
+                // Must be disabled when using MAS.
+                password_config: {
+                    enabled: false,
+                },
+                // Must be disabled when using MAS.
+                enable_registration: false,
+            });
+        }
         return this;
     }
 

package/tsconfig.json

@@ -1,11 +1,16 @@
 {
     "$schema": "http://json.schemastore.org/tsconfig",
-    "extends": "../../tsconfig.json",
     "compilerOptions": {
+        "target": "esnext",
+        "lib": ["dom", "es2022", "esnext"],
+        "esModuleInterop": true,
+        "strict": true,
+        "declaration": true,
         "outDir": "lib",
         "declarationMap": true,
-        "module": "node16",
-        "moduleResolution": "node16",
+        "module": "es2022",
+        "moduleResolution": "bundler",
+        "types": [],
         "allowImportingTsExtensions": false
     },
     "include": ["src"]