@public-ui/visual-tests 4.0.0-alpha.0 → 4.0.0-alpha.10

package/README.md

@@ -1,5 +1,15 @@
 # KoliBri - Visual Tests
 
+Utilities for screenshot based regression testing of KoliBri themes.
+
+[![npm](https://img.shields.io/npm/v/@public-ui/visual-tests)](https://www.npmjs.com/package/@public-ui/components)
+[![license](https://img.shields.io/npm/l/@public-ui/visual-tests)](https://github.com/public-ui/kolibri/blob/main/LICENSE)
+[![downloads](https://img.shields.io/npm/dt/@public-ui/visual-tests)](https://www.npmjs.com/package/@public-ui/visual-tests)
+[![issues](https://img.shields.io/github/issues/public-ui/kolibri)](https://github.com/public-ui/kolibri/issues)
+[![pull requests](https://img.shields.io/github/issues-pr/public-ui/kolibri)](https://github.com/public-ui/kolibri/pulls)
+[![size](https://img.shields.io/bundlephobia/min/@public-ui/visual-tests)](https://bundlephobia.com/result?p=@public-ui/visual-tests)
+![contributors](https://img.shields.io/github/contributors/public-ui/kolibri)
+
 ## Motivation
 
 The `KoliBri` Visual Tests provide a way to add visual regression testing to **theme** modules.  
@@ -7,11 +17,23 @@ It takes screenshots of every component defined in the [React Sample App](https:
 
 ## Installation
 
+It is recommended to configure NPM via `.npmrc`:
+
+```bash
+# - npm
+engine-strict=true
+save-exact=true
+
+# - pnpm
+shamefully-hoist=true # this is required for the visual tests to work
+workspace-concurrency=1
+```
+
 You can install the `KoliBri` Visual Tests with `npm`, `pnpm` or `yarn`:
 
 ```bash
 npm i -D @public-ui/visual-tests
-pnpm i -D @public-ui/visual-tests
+pnpm i -D @public-ui/visual-tests # recommended
 yarn add -D @public-ui/visual-tests
 ```
 
@@ -21,17 +43,24 @@ Add the following npm scripts to the theme's `package.json`:
 
 ```json
 {
-    "scripts": {
-        "test": "THEME_MODULE=src/index THEME_EXPORT=THEME_NAME kolibri-visual-test",
-        "test-update": "THEME_MODULE=src/index THEME_EXPORT=THEME_NAME kolibri-visual-test --update-snapshots",
-    }
+	"scripts": {
+		"test": "THEME_MODULE=src/index THEME_EXPORT=THEME_NAME kolibri-visual-test",
+		"test:update:e2e": "THEME_MODULE=src/index THEME_EXPORT=THEME_NAME kolibri-visual-test --update-snapshots=changed"
+	}
 }
 ```
 
-* `THEME_MODULE` defines the relative path to the TypeScript module containing the the theme definitions. Without file extension. 
-* `THEME_EXPERT` defines the name of the export within the the module. (e.g., `export const THEME_NAME = {/**/};`) Defaults to `default`.  
+### Environment variables
+
+- `THEME_MODULE`: Define the relative path to the TypeScript module containing the theme definitions. Without file extension.
+- `THEME_EXPERT`: Define the name of the export within the module. (e.g., `export const THEME_NAME = {/**/};`) Defaults to `default`.
+- `KOLIBRI_VISUAL_TESTS_TIMEOUT`: Define the Playwright [test timeout](https://playwright.dev/docs/test-timeouts).
+- `KOLIBRI_VISUAL_TESTS_EXPECT_TIMEOUT`: Define the Playwright [expect timeout](https://playwright.dev/docs/test-timeouts).
+- `KOLIBRI_VISUAL_TESTS_COLOR_SCHEME`: Choose the [CSS color scheme](https://developer.mozilla.org/docs/Web/CSS/@media/prefers-color-scheme) for the browser context. Supported values are `light` (default) and `dark`.
 
 Run the tests with `npm test`. The first time, this will create a new folder `snapshots` which is supposed to be committed to the repository.
 In the following runs, new screenshots will be compared to this reference.
 
-To update the reference screenshots call `npm run test-update`.
+To update the reference screenshots call `npm run test:update`.
+
+For details on theming see the [default theme README](../../themes/default/README.md).

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@public-ui/visual-tests",
-  "version": "4.0.0-alpha.0",
+  "version": "4.0.0-alpha.10",
   "license": "EUPL-1.2",
   "homepage": "https://public-ui.github.io",
   "repository": {
@@ -15,6 +15,9 @@
     "name": "Informationstechnikzentrum Bund",
     "email": "kolibri@itzbund.de"
   },
+  "engines": {
+    "pnpm": "^10"
+  },
   "type": "module",
   "sideEffects": false,
   "description": "Provides utility to run visual regression tests for themes.",
@@ -22,19 +25,24 @@
     "kolibri-visual-test": "src/index.js"
   },
   "dependencies": {
-    "@playwright/test": "1.49.0",
-    "axe-playwright": "2.0.3",
-    "portfinder": "1.0.32",
-    "serve": "14.2.4",
-    "@public-ui/sample-react": "4.0.0-alpha.0"
+    "@axe-core/playwright": "4.11.0",
+    "axe-html-reporter": "2.2.11",
+    "portfinder": "1.0.38",
+    "serve": "14.2.5",
+    "@public-ui/sample-react": "4.0.0-alpha.10"
   },
   "devDependencies": {
-    "@babel/eslint-parser": "7.25.9",
-    "@babel/plugin-syntax-import-attributes": "7.26.0",
-    "@babel/preset-env": "7.26.0",
+    "@babel/eslint-parser": "7.28.5",
+    "@babel/plugin-syntax-import-attributes": "7.27.1",
+    "@babel/preset-env": "7.28.5",
+    "@playwright/test": "1.57.0",
     "eslint": "8.57.1",
-    "knip": "5.37.1",
-    "prettier": "3.3.3"
+    "knip": "5.73.4",
+    "prettier": "3.7.4",
+    "prettier-plugin-organize-imports": "4.3.0"
+  },
+  "peerDependencies": {
+    "@playwright/test": "1.57.0"
   },
   "files": [
     "playwright.config.js",
@@ -43,8 +51,8 @@
   ],
   "scripts": {
     "format": "prettier --check src",
-    "lint": "eslint src",
-    "unused": "knip",
-    "postinstall": "pnpm exec playwright install"
+    "lint": "pnpm lint:eslint",
+    "lint:eslint": "eslint src",
+    "unused": "knip"
   }
 }

package/playwright.config.js

@@ -2,19 +2,34 @@ import { defineConfig, devices } from '@playwright/test';
 import * as path from 'path';
 import * as process from 'process';
 
-const PORT = Number(process.env.KOLIBRI_VISUAL_TEST_PORT);
-const URL = `http://localhost:${PORT}`;
+// Validate and set ENVs
+const PORT = parseInt(process.env.KOLIBRI_VISUAL_TEST_PORT || '', 10);
+const BASE_URL = `http://localhost:${PORT}`;
 
-console.log('Serving React Sample app:', URL);
+const CWD = process.env.KOLIBRI_CWD ?? '';
+const TIMEOUT = parseInt(process.env.KOLIBRI_VISUAL_TESTS_TIMEOUT || '15000', 10);
+const EXPECT_TIMEOUT = parseInt(process.env.KOLIBRI_VISUAL_TESTS_EXPECT_TIMEOUT || '5000', 10);
+const BUILD_PATH = process.env.KOLIBRI_VISUAL_TESTS_BUILD_PATH ?? '';
+const THEME = (process.env.THEME_EXPORT || 'default').toLocaleLowerCase();
+
+const VALID_COLOR_SCHEMES = ['light', 'dark'];
+const colorSchemeInput = process.env.KOLIBRI_VISUAL_TESTS_COLOR_SCHEME;
+const colorSchema = (colorSchemeInput || 'light').toLocaleLowerCase();
+
+if (!VALID_COLOR_SCHEMES.includes(colorSchema)) {
+	throw new Error(
+		`Environment variable KOLIBRI_VISUAL_TESTS_COLOR_SCHEME must be one of "${VALID_COLOR_SCHEMES.join('", "')}" (received "${colorSchemeInput}").`,
+	);
+}
 
 /**
  * See https://playwright.dev/docs/test-configuration.
  */
 export default defineConfig({
 	testDir: './tests',
-	snapshotDir: path.join(process.env.KOLIBRI_CWD ?? '', 'snapshots'),
+	snapshotDir: path.join(CWD, 'snapshots'),
 	// snapshotPathTemplate: '',
-	outputDir: path.join(process.env.KOLIBRI_CWD ?? '', 'test-results'),
+	outputDir: path.join(CWD, 'test-results'),
 	/* Run tests in files in parallel */
 	fullyParallel: true,
 	/* Fail the build on CI if you accidentally left test.only in the source code. */
@@ -24,18 +39,22 @@ export default defineConfig({
 	/* Opt out of parallel tests on CI. */
 	workers: process.env.CI ? 1 : undefined,
 	/* Allow to override the expectation timeout for slow environments */
-	timeout: process.env.KOLIBRI_VISUAL_TESTS_TIMEOUT ? Number(process.env.KOLIBRI_VISUAL_TESTS_TIMEOUT) : undefined,
+	timeout: TIMEOUT,
 	/* Reporter to use. See https://playwright.dev/docs/test-reporters */
 	reporter: 'line',
 	/* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
 	use: {
 		/* Base URL to use in actions like `await page.goto('/')`. */
-		baseURL: URL,
+		baseURL: BASE_URL,
 
 		/* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
 		trace: 'on-first-retry',
 	},
 
+	expect: {
+		timeout: EXPECT_TIMEOUT,
+	},
+
 	/* Configure projects for major browsers */
 	projects: [
 		// {
@@ -55,8 +74,9 @@ export default defineConfig({
 	/* Run your local dev server before starting the tests */
 	webServer: {
 		command: `npx serve -p ${PORT}`,
-		cwd: path.resolve(process.env.KOLIBRI_VISUAL_TESTS_BUILD_PATH),
-		url: URL,
+		cwd: path.resolve(BUILD_PATH),
+		url: BASE_URL,
 		reuseExistingServer: false,
 	},
+	snapshotPathTemplate: `{snapshotDir}/theme-${THEME}${colorSchema === 'light' ? '' : `-${colorSchema}`}/{arg}-{projectName}-{platform}{ext}`,
 });

package/src/index.js

@@ -1,12 +1,14 @@
-import child_process from 'node:child_process';
-import path from 'node:path';
-import { fileURLToPath, pathToFileURL } from 'url';
 import * as crypto from 'crypto';
 import * as fs from 'fs';
+import { readFile } from 'fs/promises';
+import child_process from 'node:child_process';
+import os from 'node:os';
+import path from 'node:path';
 import portfinder from 'portfinder';
 import * as process from 'process';
+import { fileURLToPath, pathToFileURL } from 'url';
 
-const tempDir = process.env.RUNNER_TEMP || process.env.TMPDIR; // TODO: Check on Windows
+const tempDir = process.env.RUNNER_TEMP || process.env.TMPDIR || os.tmpdir(); // TODO: Check on Windows
 
 if (!process.env.THEME_MODULE) {
 	throw new Error('Environment variable THEME_MODULE not specified.');
@@ -18,15 +20,8 @@ if (!tempDir) {
 process.env.THEME_MODULE = path.join(process.cwd(), process.env.THEME_MODULE); // Use current working directory (i.e. the theme folder) to complete module path
 const visualsTestModulePath = fileURLToPath(new URL('..', import.meta.url));
 const binaryPath = fileURLToPath(new URL('../node_modules/.bin', import.meta.url));
-
-let sampleReactPath = '../node_modules/@public-ui/sample-react';
-let backSteps = ``;
-let workingDir = null;
-do {
-	const url = new URL(backSteps + sampleReactPath, import.meta.url);
-	workingDir = fileURLToPath(url);
-	backSteps += `../`;
-} while (!fs.existsSync(workingDir) && path.resolve(process.cwd(), backSteps) !== '/');
+const sampleReactPackageJsonPath = import.meta.resolve('@public-ui/sample-react/package.json');
+const workingDir = fileURLToPath(path.dirname(sampleReactPackageJsonPath));
 
 if (!fs.existsSync(workingDir)) {
 	throw new Error('Could not find React Sample App package. Please install it with "npm install @public-ui/sample-react".');
@@ -35,19 +30,25 @@ if (!fs.existsSync(workingDir)) {
 const buildPath = path.join(tempDir, `kolibri-visual-testing-build-${crypto.randomUUID()}`);
 const rawPackageJsonPath = new URL(path.join(workingDir, 'package.json'), import.meta.url).href;
 const packageJsonPath = process.platform === 'win32' ? pathToFileURL(rawPackageJsonPath) : rawPackageJsonPath;
-const packageJson = await import(packageJsonPath, {
-	assert: { type: 'json' },
-});
+const packageJsonContent = await readFile(new URL(packageJsonPath, import.meta.url), 'utf8');
+const packageJson = JSON.parse(packageJsonContent);
 
 console.log(`
-Building React Sample App (v${packageJson?.default?.version ?? '#.#.#'}) …`);
+Building React Sample App (v${packageJson?.version ?? '#.#.#'}) …`);
 
-child_process.spawnSync('pnpm', ['run', 'build', '--', `--output-path="${buildPath}"`], {
+const buildResult = child_process.spawnSync('pnpm', ['run', 'build', `--outDir="${buildPath}"`], {
 	cwd: workingDir,
 	encoding: 'utf-8',
 	shell: true,
 });
 
+if (buildResult.status !== 0) {
+	console.log('Build status:', buildResult.status);
+	console.log('Build stdout:', buildResult.stdout);
+	console.log('Build stderr:', buildResult.stderr);
+	console.log('Build error:', buildResult.error);
+}
+
 console.log(`React Sample App build finished. Directory:`, buildPath);
 
 void (async () => {

package/tests/axe-snapshots.spec.js

@@ -1,7 +1,13 @@
+import AxeBuilder from '@axe-core/playwright';
 import { test } from '@playwright/test';
-import { checkA11y, injectAxe } from 'axe-playwright';
+import axeHtmlReporter from 'axe-html-reporter';
+import process from 'process';
 import { ROUTES } from './sample-app.routes.js';
 
+const { createHtmlReport } = axeHtmlReporter;
+
+const AXE_TAGS = ['best-practices', 'wcag2a', 'wcag2aa', 'wcag21aa'];
+
 const themeName = (process.env.THEME_EXPORT || 'default').toLocaleLowerCase();
 const rename = (snapshotName) => {
 	const result = snapshotName
@@ -24,6 +30,25 @@ const rename = (snapshotName) => {
 	return result;
 };
 
+const sanitizeRouteForReport = (route) => route.replace(/[/?]/g, '-').replace(/^-+/, '') || 'root';
+
+const buildReportOptions = (testInfo, route) => ({
+	projectKey: `axe-${themeName}`,
+	reportFileName: `${sanitizeRouteForReport(route)}.html`,
+	outputDirPath: rename(testInfo.outputDir),
+	outputDir: `axe-${themeName}`,
+});
+
+const logViolations = (route, violations) => {
+	if (!violations?.length) {
+		return;
+	}
+	console.warn(`Axe found ${violations.length} violation(s) on ${route}`);
+	for (const violation of violations) {
+		console.warn(`- ${violation.id}: ${violation.help} (${violation.nodes.length} nodes)`);
+	}
+};
+
 // https://playwright.dev/docs/emulation
 test.use({
 	colorScheme: 'light',
@@ -36,44 +61,40 @@ test.use({
 	},
 });
 
-const blocklist = [];
-
 ROUTES.forEach((options, route) => {
-	if (blocklist.includes(route)) {
+	// Skip unnecessary axe tests
+	if (options?.axe?.skip === true || process.argv.includes('--update-snapshots')) {
 		return;
 	}
 	test(`snapshot for ${route}`, async ({ page }, testInfo) => {
-		const outputPath = rename(testInfo.outputDir);
-		await page.goto(`/#${route}?hideMenus`, { waitUntil: 'networkidle' });
-		if (options?.viewportSize) {
-			await page.setViewportSize(options.viewportSize);
+		const hideMenusParam = `${route.includes('?') ? '&' : '?'}hideMenus`;
+		await page.goto(`/#${route}${hideMenusParam}`);
+		await page.waitForLoadState('networkidle');
+		await page.addStyleTag({
+			content: `
+				* {
+					transition: none !important;
+					animation: none !important;
+				}
+			`,
+		});
+		if (options?.snapshot?.viewportSize) {
+			await page.setViewportSize(options?.snapshot?.viewportSize);
 		}
-		if (options?.waitForTimeout) {
-			await page.waitForTimeout(options.waitForTimeout);
+		if (options?.snapshot?.waitForTimeout) {
+			await page.waitForTimeout(options?.snapshot?.waitForTimeout);
 		}
-		await injectAxe(page);
-		await checkA11y(
-			page,
-			undefined,
-			{
-				axeOptions: {
-					runOnly: {
-						type: 'tag',
-						values: ['best-practices', 'wcag2a', 'wcag2aa', 'wcag21aa'],
-					},
-				},
-				detailedReport: true,
-				detailedReportOptions: {
-					html: true,
-				},
-			},
-			options?.axe?.skipFailures ?? true,
-			'html',
-			{
-				outputDirPath: outputPath.replace(/\/[^/]+$/, ''),
-				outputDir: `axe-${themeName}`,
-				reportFileName: `${route.replace('/', '-')}.html`,
-			},
-		);
+
+		const builder = new AxeBuilder({ page }).withTags(AXE_TAGS);
+		const results = await builder.analyze();
+		await testInfo.attach('axe-results', {
+			body: JSON.stringify(results, null, 2),
+			contentType: 'application/json',
+		});
+		createHtmlReport({
+			results,
+			options: buildReportOptions(testInfo, route),
+		});
+		logViolations(route, results.violations);
 	});
 });