@element-hq/element-web-playwright-common 2.3.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Dockerfile +14 -8
- package/README.md +2 -2
- package/docker-entrypoint.sh +1 -8
- package/lib/expect/axe.d.ts +1 -1
- package/lib/expect/axe.d.ts.map +1 -1
- package/lib/expect/screenshot.d.ts +1 -1
- package/lib/expect/screenshot.d.ts.map +1 -1
- package/lib/fixtures/axe.d.ts +2 -2
- package/lib/fixtures/axe.d.ts.map +1 -1
- package/lib/fixtures/services.d.ts.map +1 -1
- package/lib/fixtures/services.js +2 -22
- package/lib/fixtures/user.d.ts +2 -2
- package/lib/fixtures/user.d.ts.map +1 -1
- package/lib/flaky-reporter.d.ts +24 -0
- package/lib/flaky-reporter.d.ts.map +1 -0
- package/lib/flaky-reporter.js +153 -0
- package/lib/stale-screenshot-reporter.d.ts +2 -1
- package/lib/stale-screenshot-reporter.d.ts.map +1 -1
- package/lib/stale-screenshot-reporter.js +13 -5
- package/lib/testcontainers/index.d.ts +2 -1
- package/lib/testcontainers/index.d.ts.map +1 -1
- package/lib/testcontainers/index.js +2 -1
- package/lib/testcontainers/mas.d.ts +5 -2
- package/lib/testcontainers/mas.d.ts.map +1 -1
- package/lib/testcontainers/mas.js +14 -2
- package/lib/testcontainers/postgres.d.ts +5 -0
- package/lib/testcontainers/postgres.d.ts.map +1 -0
- package/lib/testcontainers/postgres.js +31 -0
- package/lib/testcontainers/synapse.d.ts +6 -0
- package/lib/testcontainers/synapse.d.ts.map +1 -1
- package/lib/testcontainers/synapse.js +17 -1
- package/package.json +9 -10
- package/playwright-screenshots.sh +33 -126
- package/project.json +38 -0
- package/src/fixtures/services.ts +3 -22
- package/src/flaky-reporter.ts +188 -0
- package/src/stale-screenshot-reporter.ts +14 -4
- package/src/testcontainers/index.ts +2 -0
- package/src/testcontainers/mas.ts +21 -0
- package/src/testcontainers/postgres.ts +40 -0
- package/src/testcontainers/synapse.ts +24 -1
- package/tsconfig.json +8 -3
|
@@ -136,6 +136,7 @@ const DEFAULT_CONFIG = {
|
|
|
136
136
|
},
|
|
137
137
|
room_list_publication_rules: [{ action: "allow" }],
|
|
138
138
|
modules: [],
|
|
139
|
+
matrix_authentication_service: undefined,
|
|
139
140
|
};
|
|
140
141
|
/**
|
|
141
142
|
* A Synapse testcontainer
|
|
@@ -217,7 +218,22 @@ export class SynapseContainer extends GenericContainer {
|
|
|
217
218
|
return this;
|
|
218
219
|
}
|
|
219
220
|
withMatrixAuthenticationService(mas) {
|
|
220
|
-
|
|
221
|
+
if (mas) {
|
|
222
|
+
this.mas = mas;
|
|
223
|
+
this.withConfig({
|
|
224
|
+
matrix_authentication_service: {
|
|
225
|
+
enabled: true,
|
|
226
|
+
endpoint: `http://${mas.getHostname()}:8080/`,
|
|
227
|
+
secret: mas.sharedSecret,
|
|
228
|
+
},
|
|
229
|
+
// Must be disabled when using MAS.
|
|
230
|
+
password_config: {
|
|
231
|
+
enabled: false,
|
|
232
|
+
},
|
|
233
|
+
// Must be disabled when using MAS.
|
|
234
|
+
enable_registration: false,
|
|
235
|
+
});
|
|
236
|
+
}
|
|
221
237
|
return this;
|
|
222
238
|
}
|
|
223
239
|
async start() {
|
package/package.json
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@element-hq/element-web-playwright-common",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "
|
|
4
|
+
"version": "3.0.0",
|
|
5
5
|
"license": "SEE LICENSE IN README.md",
|
|
6
6
|
"repository": {
|
|
7
7
|
"type": "git",
|
|
8
|
-
"url": "git+https://github.com/element-hq/element-
|
|
9
|
-
"directory": "packages/
|
|
8
|
+
"url": "git+https://github.com/element-hq/element-web.git",
|
|
9
|
+
"directory": "packages/playwright-common"
|
|
10
10
|
},
|
|
11
11
|
"author": "element-hq",
|
|
12
12
|
"engines": {
|
|
@@ -18,15 +18,14 @@
|
|
|
18
18
|
"playwright-screenshots": "playwright-screenshots.sh"
|
|
19
19
|
},
|
|
20
20
|
"scripts": {
|
|
21
|
-
"
|
|
22
|
-
"lint:types": "tsc --noEmit"
|
|
23
|
-
"lint:codestyle": "echo 'handled by lint:eslint'",
|
|
24
|
-
"test": "echo No tests for @element-hq/element-web-playwright-common"
|
|
21
|
+
"prepack": "nx build:playwright",
|
|
22
|
+
"lint:types": "tsc --noEmit"
|
|
25
23
|
},
|
|
26
24
|
"devDependencies": {
|
|
27
25
|
"@element-hq/element-web-module-api": "*",
|
|
28
26
|
"@types/lodash-es": "^4.17.12",
|
|
29
|
-
"typescript": "^5.8.2"
|
|
27
|
+
"typescript": "^5.8.2",
|
|
28
|
+
"wait-on": "^9.0.4"
|
|
30
29
|
},
|
|
31
30
|
"dependencies": {
|
|
32
31
|
"@axe-core/playwright": "^4.10.1",
|
|
@@ -40,7 +39,7 @@
|
|
|
40
39
|
},
|
|
41
40
|
"peerDependencies": {
|
|
42
41
|
"@element-hq/element-web-module-api": "*",
|
|
43
|
-
"@playwright/test": "
|
|
44
|
-
"playwright-core": "
|
|
42
|
+
"@playwright/test": "catalog:",
|
|
43
|
+
"playwright-core": "catalog:"
|
|
45
44
|
}
|
|
46
45
|
}
|
|
@@ -6,137 +6,44 @@ set -e
|
|
|
6
6
|
SCRIPT_PATH=$(readlink -f "$0")
|
|
7
7
|
SCRIPT_DIR=$(dirname "$SCRIPT_PATH")
|
|
8
8
|
|
|
9
|
-
|
|
9
|
+
function build_image() {
|
|
10
|
+
local IMAGE_NAME="$1"
|
|
10
11
|
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
build_image() {
|
|
14
|
-
echo "Building $IMAGE_NAME image in $SCRIPT_DIR"
|
|
15
|
-
|
|
16
|
-
# Check the playwright version
|
|
17
|
-
PM=$(cat package.json | jq -r '.packageManager')
|
|
18
|
-
if [[ $PM == "pnpm@"* ]]; then
|
|
19
|
-
PW_VERSION=$(pnpm list @playwright/test --depth=0 --json | jq -r '.[].devDependencies["@playwright/test"].version')
|
|
20
|
-
else
|
|
21
|
-
PW_VERSION=$(yarn list --pattern @playwright/test --depth=0 --json --non-interactive --no-progress | jq -r '.data.trees[].name | split("@") | last')
|
|
22
|
-
fi
|
|
23
|
-
echo "with Playwright version $PW_VERSION"
|
|
24
|
-
|
|
25
|
-
# Build image
|
|
26
|
-
docker build -t "$IMAGE_NAME" --build-arg "PLAYWRIGHT_VERSION=$PW_VERSION" "$SCRIPT_DIR"
|
|
12
|
+
echo "Building $IMAGE_NAME image in $SCRIPT_DIR"
|
|
13
|
+
docker build -t "$IMAGE_NAME" --build-arg "PLAYWRIGHT_VERSION=${IMAGE_NAME#*:}" "$SCRIPT_DIR"
|
|
27
14
|
}
|
|
28
15
|
|
|
29
|
-
|
|
30
|
-
case "$DOCKER_HOST" in
|
|
31
|
-
unix://*)
|
|
32
|
-
docker_sock="${DOCKER_HOST:7}"
|
|
33
|
-
;;
|
|
34
|
-
"")
|
|
35
|
-
docker_sock="/var/run/docker.sock"
|
|
36
|
-
;;
|
|
37
|
-
*)
|
|
38
|
-
echo "$0: unsupported DOCKER_HOST setting '${DOCKER_HOST}'" >&2
|
|
39
|
-
exit 1;
|
|
40
|
-
;;
|
|
41
|
-
esac
|
|
42
|
-
|
|
43
|
-
RUN_ARGS=(
|
|
44
|
-
--rm
|
|
45
|
-
--network host
|
|
46
|
-
# Pass BASE_URL and CI environment variables to the container
|
|
47
|
-
-e BASE_URL
|
|
48
|
-
-e CI
|
|
49
|
-
# Bind mount the working directory into the container
|
|
50
|
-
-v $(pwd):/work/
|
|
51
|
-
# Bind mount the docker socket so we can run docker commands from the container
|
|
52
|
-
-v "${docker_sock}":/var/run/docker.sock
|
|
53
|
-
# Bind mount /tmp so we can store temporary files
|
|
54
|
-
-v /tmp/:/tmp/
|
|
55
|
-
-it
|
|
56
|
-
)
|
|
57
|
-
|
|
58
|
-
DEFAULT_ARGS=(--grep @screenshot)
|
|
59
|
-
LINK_MODULES=true
|
|
60
|
-
|
|
61
|
-
# Some arguments to customise behaviour so the same script / image can be
|
|
62
|
-
# re-used for other screenshot generation.
|
|
63
|
-
while [[ $# -gt 0 ]]; do
|
|
64
|
-
case "$1" in
|
|
65
|
-
# Mounts a separate node_modules directory from a docker volume in the container.
|
|
66
|
-
# Must be used if executing something that requires native node modules
|
|
67
|
-
# It's a volume rather than a directory because otherwise things tend to start picking up
|
|
68
|
-
# files from it in the native environment and break.
|
|
69
|
-
--with-node-modules)
|
|
70
|
-
mount_param="type=volume,src=ew-docker-node-modules,dst=/work/node_modules"
|
|
71
|
-
# podman doesn't support `volume-nocopy`
|
|
72
|
-
if [ -z "$docker_is_podman" ]; then mount_param+=",volume-nocopy"; fi
|
|
73
|
-
RUN_ARGS+=(--mount "${mount_param}" -e YARN_INSTALL=true)
|
|
74
|
-
shift
|
|
75
|
-
;;
|
|
76
|
-
# Disables the automatic detection & linking of node_modules which can clash with developer tooling e.g. pnpm-link
|
|
77
|
-
--no-link-modules)
|
|
78
|
-
LINK_MODULES=false
|
|
79
|
-
shift
|
|
80
|
-
;;
|
|
81
|
-
# Sets a different entrypoint (in which case the default arguments to the script will be ignored)
|
|
82
|
-
--entrypoint)
|
|
83
|
-
shift
|
|
84
|
-
RUN_ARGS+=(--entrypoint "$1")
|
|
85
|
-
DEFAULT_ARGS=()
|
|
86
|
-
shift
|
|
87
|
-
;;
|
|
88
|
-
*)
|
|
89
|
-
break
|
|
90
|
-
;;
|
|
91
|
-
esac
|
|
92
|
-
done
|
|
93
|
-
|
|
94
|
-
build_image
|
|
16
|
+
WS_PORT=3000
|
|
95
17
|
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
SYMLINKS=$(find . -maxdepth 2 -type l -not -path "./.bin/*")
|
|
100
|
-
popd > /dev/null
|
|
101
|
-
for LINK in $SYMLINKS; do
|
|
102
|
-
TARGET=$(readlink -f "node_modules/$LINK") || true
|
|
103
|
-
if [ -d "$TARGET" ]; then
|
|
104
|
-
if [ -n "$docker_is_podman" ]; then
|
|
105
|
-
echo -e "\033[31m" >&2
|
|
106
|
-
cat <<'EOF' >&2
|
|
107
|
-
WARNING: `node_modules` contains symlinks, and the support for this in
|
|
108
|
-
`playwright-screenshots.sh` is broken under podman due to
|
|
109
|
-
https://github.com/containers/podman/issues/25947.
|
|
18
|
+
# Check the playwright version
|
|
19
|
+
PW_VERSION=$(pnpm --silent -- playwright --version | awk '{print $2}')
|
|
20
|
+
IMAGE_NAME="ghcr.io/element-hq/element-web/playwright-server:$PW_VERSION"
|
|
110
21
|
|
|
111
|
-
If
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
done
|
|
22
|
+
# If the image exists in the repository, pull it; otherwise, build it.
|
|
23
|
+
#
|
|
24
|
+
# (This explicit test gives the user clearer progress info than just
|
|
25
|
+
# `docker pull 2>/dev/null || build_image`.)
|
|
26
|
+
if docker manifest inspect "$IMAGE_NAME" &>/dev/null; then
|
|
27
|
+
docker pull "$IMAGE_NAME"
|
|
28
|
+
else
|
|
29
|
+
build_image "$IMAGE_NAME"
|
|
120
30
|
fi
|
|
121
31
|
|
|
122
|
-
#
|
|
123
|
-
|
|
124
|
-
#
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
#
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
# find the running Ryuk instance and connect to it rather than start a new one.
|
|
137
|
-
#
|
|
138
|
-
# [1] https://testcontainers.com/
|
|
139
|
-
# [2] https://github.com/testcontainers/moby-ryuk
|
|
140
|
-
docker run -d --rm --label org.testcontainers.ryuk=true -v "${docker_sock}":/var/run/docker.sock -p 8080 --name="playwright-ryuk" testcontainers/ryuk:0.14.0
|
|
32
|
+
# Start the playwright-server in docker
|
|
33
|
+
CONTAINER=$(docker run --network=host -v /tmp:/tmp --rm -d -e PORT="$WS_PORT" "$IMAGE_NAME")
|
|
34
|
+
# Set up an exit trap to clean up the docker container
|
|
35
|
+
clean_up() {
|
|
36
|
+
ARG=$?
|
|
37
|
+
echo "Stopping playwright-server"
|
|
38
|
+
docker stop "$CONTAINER" > /dev/null
|
|
39
|
+
exit $ARG
|
|
40
|
+
}
|
|
41
|
+
trap clean_up EXIT
|
|
42
|
+
|
|
43
|
+
# Wait for playwright-server to be ready
|
|
44
|
+
echo "Waiting for playwright-server"
|
|
45
|
+
pnpm --dir "$SCRIPT_DIR" exec wait-on "tcp:$WS_PORT"
|
|
141
46
|
|
|
142
|
-
|
|
47
|
+
# Run the test we were given, setting PW_TEST_CONNECT_WS_ENDPOINT accordingly
|
|
48
|
+
echo "Running '$@'"
|
|
49
|
+
PW_TEST_CONNECT_WS_ENDPOINT="http://localhost:$WS_PORT" "$@"
|
package/project.json
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "../../node_modules/nx/schemas/project-schema.json",
|
|
3
|
+
"projectType": "library",
|
|
4
|
+
"root": "packages/playwright-common",
|
|
5
|
+
"targets": {
|
|
6
|
+
"build:playwright": {
|
|
7
|
+
"cache": true,
|
|
8
|
+
"command": "tsc",
|
|
9
|
+
"inputs": ["src"],
|
|
10
|
+
"outputs": ["{projectRoot}/lib"],
|
|
11
|
+
"options": { "cwd": "packages/playwright-common" }
|
|
12
|
+
},
|
|
13
|
+
"docker:prebuild": {
|
|
14
|
+
"cache": true,
|
|
15
|
+
"command": "echo PLAYWRIGHT_VERSION=$(pnpm --silent -- playwright --version | awk '{print $2}') > .env.docker:build",
|
|
16
|
+
"inputs": [{ "runtime": "pnpm --silent -- playwright --version" }],
|
|
17
|
+
"outputs": ["{projectRoot}/.env.docker:build"],
|
|
18
|
+
"options": { "cwd": "packages/playwright-common" }
|
|
19
|
+
},
|
|
20
|
+
"docker:build": {
|
|
21
|
+
"executor": "@nx-tools/nx-container:build",
|
|
22
|
+
"dependsOn": ["docker:prebuild"],
|
|
23
|
+
"options": {
|
|
24
|
+
"load": true,
|
|
25
|
+
"engine": "docker",
|
|
26
|
+
"platforms": ["linux/amd64", "linux/arm64"],
|
|
27
|
+
"provenance": "true",
|
|
28
|
+
"sbom": true,
|
|
29
|
+
"build-args": ["PLAYWRIGHT_VERSION=$PLAYWRIGHT_VERSION"],
|
|
30
|
+
"context": "{projectRoot}",
|
|
31
|
+
"metadata": {
|
|
32
|
+
"images": ["ghcr.io/element-hq/element-web/playwright-server"],
|
|
33
|
+
"tags": ["type=ref,event=branch", "type=raw,enable={{is_default_branch}},value=$PLAYWRIGHT_VERSION"]
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
}
|
package/src/fixtures/services.ts
CHANGED
|
@@ -7,7 +7,7 @@ Please see LICENSE files in the repository root for full details.
|
|
|
7
7
|
|
|
8
8
|
import { type MailpitClient } from "mailpit-api";
|
|
9
9
|
import { Network, type StartedNetwork } from "testcontainers";
|
|
10
|
-
import {
|
|
10
|
+
import { type StartedPostgreSqlContainer } from "@testcontainers/postgresql";
|
|
11
11
|
|
|
12
12
|
import {
|
|
13
13
|
type SynapseConfig,
|
|
@@ -22,6 +22,7 @@ import { Logger } from "../utils/logger.js";
|
|
|
22
22
|
// We want to avoid using `mergeTests` in index.ts because it drops useful type information about the fixtures. Instead,
|
|
23
23
|
// we add `axe` into our fixture suite by using its `test` as a base, so that there is a linear hierarchy.
|
|
24
24
|
import { test as base } from "./axe.js";
|
|
25
|
+
import { makePostgres } from "../testcontainers/postgres.js";
|
|
25
26
|
|
|
26
27
|
/**
|
|
27
28
|
* Test-scoped fixtures available in the test
|
|
@@ -101,27 +102,7 @@ export const test = base.extend<TestFixtures, WorkerOptions & Services>({
|
|
|
101
102
|
],
|
|
102
103
|
postgres: [
|
|
103
104
|
async ({ logger, network }, use) => {
|
|
104
|
-
const container = await
|
|
105
|
-
.withNetwork(network)
|
|
106
|
-
.withNetworkAliases("postgres")
|
|
107
|
-
.withLogConsumer(logger.getConsumer("postgres"))
|
|
108
|
-
.withTmpFs({
|
|
109
|
-
"/dev/shm/pgdata/data": "",
|
|
110
|
-
})
|
|
111
|
-
.withEnvironment({
|
|
112
|
-
PG_DATA: "/dev/shm/pgdata/data",
|
|
113
|
-
})
|
|
114
|
-
.withCommand([
|
|
115
|
-
"-c",
|
|
116
|
-
"shared_buffers=128MB",
|
|
117
|
-
"-c",
|
|
118
|
-
`fsync=off`,
|
|
119
|
-
"-c",
|
|
120
|
-
`synchronous_commit=off`,
|
|
121
|
-
"-c",
|
|
122
|
-
"full_page_writes=off",
|
|
123
|
-
])
|
|
124
|
-
.start();
|
|
105
|
+
const container = await makePostgres(network, logger);
|
|
125
106
|
await use(container);
|
|
126
107
|
await container.stop();
|
|
127
108
|
},
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
/*
|
|
2
|
+
Copyright 2024 New Vector Ltd.
|
|
3
|
+
Copyright 2024 The Matrix.org Foundation C.I.C.
|
|
4
|
+
|
|
5
|
+
SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Commercial
|
|
6
|
+
Please see LICENSE files in the repository root for full details.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Flaky test reporter, creating & updating GitHub issues
|
|
11
|
+
* Only intended to run from within GitHub Actions
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import type { Reporter, TestCase } from "@playwright/test/reporter";
|
|
15
|
+
|
|
16
|
+
const REPO = "element-hq/element-web";
|
|
17
|
+
const LABEL = "Z-Flaky-Test";
|
|
18
|
+
const ISSUE_TITLE_PREFIX = "Flaky playwright test: ";
|
|
19
|
+
|
|
20
|
+
type PaginationLinks = {
|
|
21
|
+
prev?: string;
|
|
22
|
+
next?: string;
|
|
23
|
+
last?: string;
|
|
24
|
+
first?: string;
|
|
25
|
+
};
|
|
26
|
+
|
|
27
|
+
const ANSI_COLOUR_REGEX = /[\u001b\u009b][[()#;?]*(?:[0-9]{1,4}(?:;[0-9]{0,4})*)?[0-9A-ORZcf-nqry=><]/g;
|
|
28
|
+
|
|
29
|
+
// We see quite a few test flakes which are caused by the app exploding
|
|
30
|
+
// so we have some magic strings we check the logs for to better track the flake with its cause
|
|
31
|
+
const SPECIAL_CASES: Record<string, string> = {
|
|
32
|
+
"ChunkLoadError": "ChunkLoadError",
|
|
33
|
+
"Unreachable code should not be executed": "Rust crypto panic",
|
|
34
|
+
"Out of bounds memory access": "Rust crypto memory error",
|
|
35
|
+
};
|
|
36
|
+
|
|
37
|
+
class FlakyReporter implements Reporter {
|
|
38
|
+
private flakes = new Map<string, TestCase[]>();
|
|
39
|
+
|
|
40
|
+
public onTestEnd(test: TestCase): void {
|
|
41
|
+
// Ignores flakes on Dendrite and Pinecone as they have their own flakes we do not track
|
|
42
|
+
if (["Dendrite", "Pinecone"].includes(test.parent.project()!.name!)) return;
|
|
43
|
+
|
|
44
|
+
if (test.outcome() === "flaky") {
|
|
45
|
+
const failures: string[] = [];
|
|
46
|
+
|
|
47
|
+
const timedOutRuns = test.results.filter((result) => result.status === "timedOut");
|
|
48
|
+
const pageLogs = timedOutRuns.flatMap((result) =>
|
|
49
|
+
result.attachments.filter((attachment) => attachment.name.startsWith("page-")),
|
|
50
|
+
);
|
|
51
|
+
|
|
52
|
+
// If a test failed due to a systemic fault then the test is not flaky, the app is, record it as such.
|
|
53
|
+
const specialCases = Object.keys(SPECIAL_CASES).filter((log) =>
|
|
54
|
+
pageLogs.some((attachment) => attachment.name.startsWith("page-") && attachment.body?.includes(log)),
|
|
55
|
+
);
|
|
56
|
+
if (specialCases.length > 0) {
|
|
57
|
+
failures.push(...specialCases.map((specialCase) => SPECIAL_CASES[specialCase]));
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
// Check for fixtures failing to set up
|
|
61
|
+
const errorMessages = timedOutRuns
|
|
62
|
+
.map((r) => r.error?.message?.replace(ANSI_COLOUR_REGEX, ""))
|
|
63
|
+
.filter(Boolean) as string[];
|
|
64
|
+
for (const error of errorMessages) {
|
|
65
|
+
if (error.startsWith("Fixture") && error.endsWith("exceeded during setup.")) {
|
|
66
|
+
failures.push(error);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
if (failures.length < 1) {
|
|
71
|
+
failures.push(`${test.location.file.split("playwright/e2e/")[1]}: ${test.title}`);
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
for (const title of failures) {
|
|
75
|
+
if (!this.flakes.has(title)) {
|
|
76
|
+
this.flakes.set(title, []);
|
|
77
|
+
}
|
|
78
|
+
this.flakes.get(title)!.push(test);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* Parse link header to retrieve pagination links
|
|
85
|
+
* @see https://docs.github.com/en/rest/using-the-rest-api/using-pagination-in-the-rest-api?apiVersion=2022-11-28#using-link-headers
|
|
86
|
+
* @param link link header from response or undefined
|
|
87
|
+
* @returns an empty object if link is undefined otherwise returns a map from type to link
|
|
88
|
+
*/
|
|
89
|
+
private parseLinkHeader(link: string): PaginationLinks {
|
|
90
|
+
/**
|
|
91
|
+
* link looks like:
|
|
92
|
+
* <https://api.github.com/repositories/1300192/issues?page=2>; rel="prev", <https://api.github.com/repositories/1300192/issues?page=4>;
|
|
93
|
+
*/
|
|
94
|
+
const map: PaginationLinks = {};
|
|
95
|
+
if (!link) return map;
|
|
96
|
+
const matches = link.matchAll(/(<(?<link>.+?)>; rel="(?<type>.+?)")/g);
|
|
97
|
+
for (const match of matches) {
|
|
98
|
+
const { link, type } = match.groups!;
|
|
99
|
+
map[type as keyof PaginationLinks] = link;
|
|
100
|
+
}
|
|
101
|
+
return map;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/**
|
|
105
|
+
* Fetch all flaky test issues that were updated since Jan-1-2024
|
|
106
|
+
* @returns A promise that resolves to a list of issues
|
|
107
|
+
*/
|
|
108
|
+
async getAllIssues(): Promise<any[]> {
|
|
109
|
+
const issues = [];
|
|
110
|
+
const { GITHUB_TOKEN, GITHUB_API_URL } = process.env;
|
|
111
|
+
// See https://docs.github.com/en/rest/issues/issues?apiVersion=2022-11-28#list-repository-issues
|
|
112
|
+
let url = `${GITHUB_API_URL}/repos/${REPO}/issues?labels=${LABEL}&state=all&per_page=100&sort=updated&since=2024-01-01`;
|
|
113
|
+
const headers = {
|
|
114
|
+
Authorization: `Bearer ${GITHUB_TOKEN}`,
|
|
115
|
+
Accept: "application / vnd.github + json",
|
|
116
|
+
};
|
|
117
|
+
while (url) {
|
|
118
|
+
// Fetch issues and add to list
|
|
119
|
+
const issuesResponse = await fetch(url, { headers });
|
|
120
|
+
const fetchedIssues = await issuesResponse.json();
|
|
121
|
+
issues.push(...fetchedIssues);
|
|
122
|
+
|
|
123
|
+
// Get the next link for fetching more results
|
|
124
|
+
const linkHeader = issuesResponse.headers.get("Link")!;
|
|
125
|
+
const parsed = this.parseLinkHeader(linkHeader);
|
|
126
|
+
url = parsed.next!;
|
|
127
|
+
}
|
|
128
|
+
return issues;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
public async onExit(): Promise<void> {
|
|
132
|
+
if (this.flakes.size === 0) {
|
|
133
|
+
console.log("No flakes found");
|
|
134
|
+
return;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
console.log("Found flakes: ");
|
|
138
|
+
for (const flake of this.flakes) {
|
|
139
|
+
console.log(flake);
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
const { GITHUB_TOKEN, GITHUB_API_URL, GITHUB_SERVER_URL, GITHUB_REPOSITORY, GITHUB_RUN_ID } = process.env;
|
|
143
|
+
if (!GITHUB_TOKEN) return;
|
|
144
|
+
|
|
145
|
+
const issues = await this.getAllIssues();
|
|
146
|
+
for (const [flake, results] of this.flakes) {
|
|
147
|
+
const title = ISSUE_TITLE_PREFIX + "`" + flake + "`";
|
|
148
|
+
const existingIssue = issues.find((issue) => issue.title === title);
|
|
149
|
+
const headers = { Authorization: `Bearer ${GITHUB_TOKEN}` };
|
|
150
|
+
const body = `${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}`;
|
|
151
|
+
|
|
152
|
+
const labels = [LABEL, ...results.map((test) => `${LABEL}-${test.parent.project()?.name}`)];
|
|
153
|
+
|
|
154
|
+
if (existingIssue) {
|
|
155
|
+
console.log(`Found issue ${existingIssue.number} for ${flake}, adding comment...`);
|
|
156
|
+
// Ensure that the test is open
|
|
157
|
+
await fetch(existingIssue.url, {
|
|
158
|
+
method: "PATCH",
|
|
159
|
+
headers,
|
|
160
|
+
body: JSON.stringify({ state: "open" }),
|
|
161
|
+
});
|
|
162
|
+
await fetch(`${existingIssue.url}/labels`, {
|
|
163
|
+
method: "POST",
|
|
164
|
+
headers,
|
|
165
|
+
body: JSON.stringify({ labels }),
|
|
166
|
+
});
|
|
167
|
+
await fetch(`${existingIssue.url}/comments`, {
|
|
168
|
+
method: "POST",
|
|
169
|
+
headers,
|
|
170
|
+
body: JSON.stringify({ body }),
|
|
171
|
+
});
|
|
172
|
+
} else {
|
|
173
|
+
console.log(`Creating new issue for ${flake}...`);
|
|
174
|
+
await fetch(`${GITHUB_API_URL}/repos/${REPO}/issues`, {
|
|
175
|
+
method: "POST",
|
|
176
|
+
headers,
|
|
177
|
+
body: JSON.stringify({
|
|
178
|
+
title,
|
|
179
|
+
body,
|
|
180
|
+
labels: [...labels],
|
|
181
|
+
}),
|
|
182
|
+
});
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
export default FlakyReporter;
|
|
@@ -25,7 +25,7 @@ export const ANNOTATION = "_screenshot";
|
|
|
25
25
|
class StaleScreenshotReporter implements Reporter {
|
|
26
26
|
private readonly snapshotRoots = new Set<string>();
|
|
27
27
|
private readonly screenshots = new Set<string>();
|
|
28
|
-
private failing =
|
|
28
|
+
private readonly failing = new Set<string>();
|
|
29
29
|
private success = true;
|
|
30
30
|
|
|
31
31
|
public onBegin(config: FullConfig): void {
|
|
@@ -36,8 +36,11 @@ class StaleScreenshotReporter implements Reporter {
|
|
|
36
36
|
|
|
37
37
|
public onTestEnd(test: TestCase): void {
|
|
38
38
|
if (!test.ok()) {
|
|
39
|
-
this.failing
|
|
39
|
+
this.failing.add(test.id);
|
|
40
|
+
return;
|
|
40
41
|
}
|
|
42
|
+
this.failing.delete(test.id); // delete if passed on re-run
|
|
43
|
+
|
|
41
44
|
for (const annotation of test.annotations) {
|
|
42
45
|
if (annotation.type === ANNOTATION && annotation.description) {
|
|
43
46
|
this.screenshots.add(annotation.description);
|
|
@@ -53,8 +56,7 @@ class StaleScreenshotReporter implements Reporter {
|
|
|
53
56
|
this.success = false;
|
|
54
57
|
}
|
|
55
58
|
|
|
56
|
-
|
|
57
|
-
if (this.failing) return;
|
|
59
|
+
private async checkStaleScreenshots(): Promise<void> {
|
|
58
60
|
if (!this.snapshotRoots.size) {
|
|
59
61
|
this.error("No snapshot directories found, did you set the snapshotDir in your Playwright config?", "");
|
|
60
62
|
return;
|
|
@@ -84,6 +86,14 @@ class StaleScreenshotReporter implements Reporter {
|
|
|
84
86
|
this.error("Stale screenshot file", screenshot);
|
|
85
87
|
}
|
|
86
88
|
}
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
public async onExit(): Promise<void> {
|
|
92
|
+
if (this.failing.size) {
|
|
93
|
+
this.error(`${this.failing.size} tests failed, skipping stale screenshot reporter.`, "");
|
|
94
|
+
} else {
|
|
95
|
+
await this.checkStaleScreenshots();
|
|
96
|
+
}
|
|
87
97
|
|
|
88
98
|
if (!this.success) {
|
|
89
99
|
process.exit(1);
|
|
@@ -6,11 +6,13 @@ Please see LICENSE files in the repository root for full details.
|
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
8
|
export { PostgreSqlContainer, StartedPostgreSqlContainer } from "@testcontainers/postgresql";
|
|
9
|
+
export { makePostgres } from "./postgres.js";
|
|
9
10
|
export type { HomeserverInstance, HomeserverContainer, StartedHomeserverContainer } from "./HomeserverContainer.js";
|
|
10
11
|
export { type SynapseConfig, SynapseContainer, StartedSynapseContainer } from "./synapse.js";
|
|
11
12
|
export {
|
|
12
13
|
type MasConfig,
|
|
13
14
|
MatrixAuthenticationServiceContainer,
|
|
14
15
|
StartedMatrixAuthenticationServiceContainer,
|
|
16
|
+
makeMas,
|
|
15
17
|
} from "./mas.js";
|
|
16
18
|
export { type MailpitClient, MailpitContainer, StartedMailpitContainer } from "./mailpit.js";
|
|
@@ -11,6 +11,7 @@ import {
|
|
|
11
11
|
type StartedTestContainer,
|
|
12
12
|
Wait,
|
|
13
13
|
type ExecResult,
|
|
14
|
+
type StartedNetwork,
|
|
14
15
|
} from "testcontainers";
|
|
15
16
|
import { type StartedPostgreSqlContainer } from "@testcontainers/postgresql";
|
|
16
17
|
import * as YAML from "yaml";
|
|
@@ -23,6 +24,7 @@ import { type Credentials } from "../utils/api.js";
|
|
|
23
24
|
// curl -sL https://element-hq.github.io/matrix-authentication-service/config.schema.json \
|
|
24
25
|
// | npx json-schema-to-typescript -o packages/element-web-playwright-common/src/testconainers/mas-config.ts
|
|
25
26
|
import type { RootConfig as MasConfig } from "./mas-config.js";
|
|
27
|
+
import type { Logger } from "../utils/logger.js";
|
|
26
28
|
|
|
27
29
|
export { type MasConfig };
|
|
28
30
|
|
|
@@ -156,6 +158,7 @@ export class MatrixAuthenticationServiceContainer extends GenericContainer {
|
|
|
156
158
|
super(image);
|
|
157
159
|
|
|
158
160
|
const initialConfig = deepCopy(DEFAULT_CONFIG);
|
|
161
|
+
initialConfig.database.host = db.getHostname();
|
|
159
162
|
initialConfig.database.username = db.getUsername();
|
|
160
163
|
initialConfig.database.password = db.getPassword();
|
|
161
164
|
|
|
@@ -205,6 +208,7 @@ export class MatrixAuthenticationServiceContainer extends GenericContainer {
|
|
|
205
208
|
await super.start(),
|
|
206
209
|
`http://localhost:${port}`,
|
|
207
210
|
this.args,
|
|
211
|
+
this.config.matrix.secret,
|
|
208
212
|
);
|
|
209
213
|
}
|
|
210
214
|
}
|
|
@@ -219,6 +223,7 @@ export class StartedMatrixAuthenticationServiceContainer extends AbstractStarted
|
|
|
219
223
|
container: StartedTestContainer,
|
|
220
224
|
public readonly baseUrl: string,
|
|
221
225
|
private readonly args: string[],
|
|
226
|
+
public readonly sharedSecret: string,
|
|
222
227
|
) {
|
|
223
228
|
super(container);
|
|
224
229
|
}
|
|
@@ -346,3 +351,19 @@ export class StartedMatrixAuthenticationServiceContainer extends AbstractStarted
|
|
|
346
351
|
await this.manage("add-email", username, address);
|
|
347
352
|
}
|
|
348
353
|
}
|
|
354
|
+
|
|
355
|
+
export async function makeMas(
|
|
356
|
+
postgres: StartedPostgreSqlContainer,
|
|
357
|
+
network: StartedNetwork,
|
|
358
|
+
logger: Logger,
|
|
359
|
+
config: Partial<MasConfig>,
|
|
360
|
+
name = "mas",
|
|
361
|
+
): Promise<StartedMatrixAuthenticationServiceContainer> {
|
|
362
|
+
const container = await new MatrixAuthenticationServiceContainer(postgres)
|
|
363
|
+
.withNetwork(network)
|
|
364
|
+
.withNetworkAliases(name)
|
|
365
|
+
.withLogConsumer(logger.getConsumer(name))
|
|
366
|
+
.withConfig(config)
|
|
367
|
+
.start();
|
|
368
|
+
return container;
|
|
369
|
+
}
|