@public-ui/visual-tests 1.7.0-rc.9 → 1.7.0

package/README.md

@@ -36,6 +36,12 @@ In the following runs, new screenshots will be compared to this reference.
 
 To update the reference screenshots call `npm run test-update`.
 
+It's recommended to add the `test-results` folder to the `.gitignore` file:
+
+```bash
+/test-results/
+```
+
 It's also recommended to automatically run the tests before packing/publishing the module:
 
 ```json

package/package.json

@@ -1,46 +1,49 @@
 {
-	"name": "@public-ui/visual-tests",
-	"version": "1.7.0-rc.9",
-	"license": "EUPL-1.2",
-	"homepage": "https://public-ui.github.io",
-	"repository": "https://github.com/public-ui/kolibri",
-	"bugs": {
-		"url": "https://github.com/public-ui/kolibri/issues",
-		"email": "kolibri@itzbund.de"
-	},
-	"author": {
-		"name": "Informationstechnikzentrum Bund",
-		"email": "kolibri@itzbund.de"
-	},
-	"type": "module",
-	"sideEffects": false,
-	"description": "Provides utility to run visual regression tests for themes.",
-	"scripts": {
-		"depcheck": "depcheck --ignores=@public-ui/sample-react,http-server,tslib",
-		"format": "prettier --check src",
-		"lint": "eslint src",
-		"unused": "knip"
-	},
-	"bin": {
-		"kolibri-visual-test": "src/index.js"
-	},
-	"dependencies": {
-		"@playwright/test": "1.37.1",
-		"@public-ui/sample-react": "1.7.0-rc.9",
-		"http-server": "14.1.1",
-		"portfinder": "1.0.32"
-	},
-	"devDependencies": {
-		"@babel/eslint-parser": "7.22.15",
-		"depcheck": "1.4.6",
-		"eslint": "8.49.0",
-		"eslint-plugin-no-loops": "0.3.0",
-		"knip": "2.24.0",
-		"prettier": "3.0.3"
-	},
-	"files": [
-		"playwright.config.js",
-		"src",
-		"tests"
-	]
-}
+  "name": "@public-ui/visual-tests",
+  "version": "1.7.0",
+  "license": "EUPL-1.2",
+  "homepage": "https://public-ui.github.io",
+  "repository": "https://github.com/public-ui/kolibri",
+  "bugs": {
+    "url": "https://github.com/public-ui/kolibri/issues",
+    "email": "kolibri@itzbund.de"
+  },
+  "author": {
+    "name": "Informationstechnikzentrum Bund",
+    "email": "kolibri@itzbund.de"
+  },
+  "type": "module",
+  "sideEffects": false,
+  "description": "Provides utility to run visual regression tests for themes.",
+  "bin": {
+    "kolibri-visual-test": "src/index.js"
+  },
+  "dependencies": {
+    "@playwright/test": "1.38.1",
+    "@public-ui/sample-react": "1.7.0",
+    "portfinder": "1.0.32",
+    "serve": "14.2.1"
+  },
+  "devDependencies": {
+    "@babel/eslint-parser": "7.22.15",
+    "@babel/plugin-syntax-import-attributes": "7.22.5",
+    "@babel/preset-env": "7.22.20",
+    "axe-playwright": "1.2.3",
+    "depcheck": "1.4.6",
+    "eslint": "8.50.0",
+    "eslint-plugin-no-loops": "0.3.0",
+    "knip": "2.30.0",
+    "prettier": "3.0.3"
+  },
+  "files": [
+    "playwright.config.js",
+    "src",
+    "tests"
+  ],
+  "scripts": {
+    "depcheck": "depcheck --ignores=@babel/plugin-syntax-import-attributes,@babel/preset-env,@public-ui/sample-react,portfinder,serve",
+    "format": "prettier --check src",
+    "lint": "eslint src",
+    "unused": "knip"
+  }
+}

package/playwright.config.js

@@ -20,7 +20,7 @@ export default defineConfig({
 	/* Fail the build on CI if you accidentally left test.only in the source code. */
 	forbidOnly: !!process.env.CI,
 	/* Retry on CI only */
-	retries: 0, // process.env.CI ? 2 : 0,
+	retries: process.env.CI ? 2 : 0,
 	/* Opt out of parallel tests on CI. */
 	workers: process.env.CI ? 1 : undefined,
 	/* Reporter to use. See https://playwright.dev/docs/test-reporters */
@@ -52,7 +52,7 @@ export default defineConfig({
 
 	/* Run your local dev server before starting the tests */
 	webServer: {
-		command: `http-server ${process.env.KOLIBRI_VISUAL_TESTS_BUILD_PATH} -p ${PORT}`,
+		command: `serve ${process.env.KOLIBRI_VISUAL_TESTS_BUILD_PATH} -p ${PORT}`,
 		url: URL,
 		reuseExistingServer: false,
 	},

package/src/index.js

@@ -19,11 +19,31 @@ if (!tempDir) {
 process.env.THEME_MODULE = path.join(process.cwd(), process.env.THEME_MODULE); // Use current working directory (i.e. the theme folder) to complete module path
 const visualsTestModulePath = fileURLToPath(new URL('..', import.meta.url));
 const binaryPath = fileURLToPath(new URL('../node_modules/.bin', import.meta.url));
+
+let sampleReactPath = '../node_modules/@public-ui/sample-react';
+let backSteps = ``;
+let workingDir = null;
+do {
+	const url = new URL(backSteps + sampleReactPath, import.meta.url);
+	workingDir = fileURLToPath(url);
+	backSteps += `../`;
+} while (!fs.existsSync(workingDir) && path.resolve(process.cwd(), backSteps) !== '/');
+
+if (!fs.existsSync(workingDir)) {
+	throw new Error('Could not find React Sample App package. Please install it with "npm install @public-ui/sample-react".');
+}
+
 const buildPath = path.join(tempDir, `kolibri-visual-testing-build-${crypto.randomUUID()}`);
+const packageJsonPath = await import(new URL(`${workingDir}/package.json`, import.meta.url), {
+	assert: { type: 'json' },
+});
+
 process.env.KOLIBRI_VISUAL_TESTS_BUILD_PATH = buildPath;
 
-console.log('Building React Sample App …');
-child_process.execFileSync(path.join(binaryPath, 'kolibri-sample-react-test-build'), [buildPath], {
+console.log(`
+Building React Sample App (v${packageJsonPath?.default?.version ?? '#.#.#'}) …`);
+child_process.execFileSync('npm', ['run', 'build', '--', `--output-path=${buildPath}`], {
+	cwd: workingDir,
 	encoding: 'utf-8',
 });
 
@@ -46,8 +66,15 @@ void (async () => {
 
 	playwright.on('exit', (code) => {
 		console.log(`Playwright test finished with exit code ${code}.`);
-		console.log('Cleaning up build folder …');
-		fs.rmSync(buildPath, { recursive: true, force: true });
+
+		if (process.env.KOLIBRI_CLEANUP === '0') {
+			console.log('Skipping cleanup up build folder.');
+			console.log(`You can serve this build with "npx serve ${buildPath}".`);
+		} else {
+			console.log('Cleaning up build folder …');
+			fs.rmSync(buildPath, { recursive: true, force: true });
+			console.log('Cleaning up finished successfully.');
+		}
 		process.exit(code ?? 1);
 	});
 })();

package/tests/axe-snapshots.spec.js

@@ -0,0 +1,82 @@
+import { test } from '@playwright/test';
+import { checkA11y, injectAxe } from 'axe-playwright';
+import path from 'path';
+import { ROUTES } from './sample-app.routes.js';
+
+const rename = (snapshotName) => {
+	const result = snapshotName
+
+		// Remove browser name from snapshot name
+		// .replace('-chromium', '')
+		// .replace('-firefox', '')
+
+		// Remove os name from snapshot name
+		// .replace('-darwin', '')
+		// .replace('-linux', '')
+		// .replace('-windows', '')
+
+		// Remove test counter from snapshot name
+		.replace('-1-', '-')
+
+		// Make different snapshot folder for different themes
+		.replace('theme-snapshots.spec.js', `axe-${process.env.THEME_EXPORT.toLocaleLowerCase()}`)
+		.replace('-snapshots', '');
+	return result;
+};
+
+// https://playwright.dev/docs/emulation
+test.use({
+	colorScheme: 'light',
+	locale: 'de-DE',
+	isMobile: false,
+	timezoneId: 'Europe/Berlin',
+	viewport: {
+		width: 800,
+		height: 0,
+	},
+});
+
+/**
+ * @todo stabilize and re-enable test
+ */
+const blocklist = [];
+
+ROUTES.forEach((options, route) => {
+	if (blocklist.includes(route)) {
+		return;
+	}
+	test(`snapshot for ${route}`, async ({ page }, testInfo) => {
+		const outputPath = rename(testInfo.outputDir);
+		await page.goto(`/#${route}?hideMenus`, { waitUntil: 'networkidle' });
+		if (options?.viewportSize) {
+			await page.setViewportSize(options.viewportSize);
+		}
+		if (options?.waitForTimeout) {
+			await page.waitForTimeout(options.waitForTimeout);
+		}
+		await injectAxe(page);
+		await checkA11y(
+			page,
+			undefined,
+			{
+				axeOptions: {
+					runOnly: {
+						type: 'tag',
+						values: ['best-practices', 'wcag2a', 'wcag2aa', 'wcag21aa'],
+					},
+				},
+				detailedReport: true,
+				detailedReportOptions: {
+					html: true,
+				},
+			},
+			options?.axe?.skipFailures ?? true,
+			'html',
+			{
+				outputDirPath: outputPath.replace(/\/[^/]+$/, ''),
+				outputDir: `axe-${process.env.THEME_EXPORT.toLocaleLowerCase()}`,
+				reportFileName: `${route.replace('/', '-')}.html`,
+			},
+		);
+	});
+});

package/tests/sample-app.routes.js

@@ -1,80 +1,387 @@
-export const ROUTES = [
-	'/handout/basic',
-	'/abbr/basic',
-	'/accordion/basic',
-	'/accordion/header',
-	'/accordion/headlines',
-	'/accordion/list',
-	'/alert/basic',
-	'/alert/card-msg',
-	'/alert/html',
-	'/avatar/basic',
-	'/badge/basic',
-	'/badge/button',
-	'/breadcrumb/basic',
-	'/button/basic',
-	'/button/hide-label',
-	'/button/icons',
-	'/button/width',
-	'/button-link/basic',
-	'/button-link/icons',
-	'/button-link/image',
-	'/button-link/target',
-	'/button-group/basic',
-	'/card/basic',
-	'/card/confirm',
-	'/card/flex',
-	'/card/selection',
-	'/details/basic',
-	'/heading/badge',
-	'/heading/basic',
-	'/heading/paragraph',
-	'/icon/basic',
-	'/indented-text/basic',
-	'/input-checkbox/basic',
-	'/input-color/basic',
-	'/input-date/basic',
-	'/input-email/basic',
-	'/input-file/basic',
-	'/input-number/basic',
-	'/input-password/basic',
-	'/input-password/show-password',
-	'/input-radio/basic',
-	'/input-radio/horizontal',
-	'/input-radio/select',
-	'/input-range/basic',
-	'/input-text/basic',
-	'/input-text/hidden-label',
-	'/input-text/blur',
-	'/input-text/focus',
-	'/link/basic',
-	'/link/icons',
-	'/link/image',
-	'/link/target',
-	'/link-button/basic',
-	'/nav/basic',
-	'/nav/active',
-	'/nav/aria-current',
-	'/nav/horizontal',
-	'/pagination/basic',
-	'/popover/basic',
-	'/progress/basic',
-	'/select/basic',
-	'/skip-nav/basic',
-	'/spin/basic',
-	'/spin/cycle',
-	'/spin/custom',
-	'/split-button/basic',
-	'/table/badge-size',
-	'/table/render-cell',
-	'/table/sort-data',
-	'/textarea/basic',
-	'/textarea/adjust-height',
-	'/textarea/disabled',
-	'/textarea/placeholder',
-	'/textarea/readonly',
-	'/textarea/resize',
-	'/textarea/rows',
-	'/version/basic',
-	'/version/context',
-];
+export const ROUTES = new Map();
+
+/**
+ * Actually we support the following options:
+ *
+ * Details: https://playwright.dev/docs/api/class-pageassertions#page-assertions-to-have-screenshot-1
+ * - fullPage: boolean (Default: true)
+ * - maxDiffPixelRatio: number (Default: undefined)
+ * - maxDiffPixels: number (Default: undefined)
+ * - threshold: number (Default: undefined)
+ * - timeout: number (Default: 0)
+ *
+ * To set the viewport size, use the following options:
+ * - viewportSize: { width, height } (Default: 800x600)
+ *
+ */
+
+ROUTES.set('handout/basic', {
+	viewportSize: {
+		width: 1920,
+		height: 1280,
+	},
+	waitForTimeout: 500,
+});
+ROUTES.set('abbr/basic', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('accordion/basic', null);
+ROUTES.set('accordion/header', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('accordion/headlines', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('accordion/list', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('alert/basic', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('alert/card-msg', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('alert/html', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('avatar/basic', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('badge/basic', null);
+ROUTES.set('badge/button', null);
+ROUTES.set('breadcrumb/basic', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('button-group/basic', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('button-link/basic', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('button-link/icons', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('button-link/image', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('button/basic', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('button/hide-label', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('button/icons', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('button/width', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('card/basic', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('card/confirm', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('card/flex', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('card/selection', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('details/basic', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('heading/badge', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('heading/basic', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('heading/paragraph', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('icon/basic', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('image/basic', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('indented-text/basic', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('input-checkbox/basic', null);
+ROUTES.set('input-checkbox/button', null);
+ROUTES.set('input-checkbox/switch', null);
+ROUTES.set('input-color/basic', null);
+ROUTES.set('input-date/basic', null);
+ROUTES.set('input-email/basic', null);
+ROUTES.set('input-file/basic', null);
+ROUTES.set('input-number/basic', null);
+ROUTES.set('input-password/basic', null);
+ROUTES.set('input-password/show-password', null);
+ROUTES.set('input-radio/basic', null);
+ROUTES.set('input-radio/horizontal', null);
+ROUTES.set('input-radio/select', null);
+ROUTES.set('input-range/basic', null);
+ROUTES.set('input-text/basic', null);
+ROUTES.set('input-text/blur', null);
+ROUTES.set('input-text/focus', null);
+ROUTES.set('kolibri/animated', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('kolibri/basic', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('kolibri/no-label', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('link-button/basic', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('link-group/basic', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('link-group/horizontal', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('link/basic', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('link/icons', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('link/image', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('link/target', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('logo/basic', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('modal/basic', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('nav/active', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('nav/aria-current', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('nav/basic', null);
+ROUTES.set('nav/horizontal', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('pagination/basic', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('popover/basic', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('progress/basic', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('quote/basic', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('quote/block', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('select/basic', null);
+ROUTES.set('skip-nav/basic', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('spin/basic', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('spin/custom', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('spin/cycle', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('split-button/basic', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('table/badge-size', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('table/render-cell', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('table/sort-data', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('tabs/basic', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('tabs/icons-only', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('textarea/adjust-height', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('textarea/basic', null);
+ROUTES.set('textarea/disabled', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('textarea/placeholder', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('textarea/readonly', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('textarea/resize', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('textarea/rows', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('textarea/with-counter', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('toast/basic', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('version/basic', {
+	axe: {
+		skipFailures: false,
+	},
+});
+ROUTES.set('version/context', {
+	axe: {
+		skipFailures: false,
+	},
+});

package/tests/theme-snapshots.spec.js

@@ -4,6 +4,7 @@ import { ROUTES } from './sample-app.routes.js';
 // https://github.com/microsoft/playwright/issues/7575#issuecomment-1288164474
 export const configureSnapshotPath =
 	() =>
+	// eslint-disable-next-line no-empty-pattern
 	({}, testInfo) => {
 		const originalSnapshotPath = testInfo.snapshotPath;
 		testInfo.snapshotPath = (snapshotName) => {
@@ -15,33 +16,55 @@ export const configureSnapshotPath =
 				// .replace('-firefox', '')
 
 				// Remove os name from snapshot name
-				.replace('-darwin', '')
-				.replace('-linux', '')
-				.replace('-windows', '')
+				// .replace('-darwin', '')
+				// .replace('-linux', '')
+				// .replace('-windows', '')
 
 				// Remove test counter from snapshot name
-				.replace('-1-', '-');
+				.replace('-1-', '-')
+
+				// Make different snapshot folder for different themes
+				.replace('theme-snapshots.spec.js', `theme-${process.env.THEME_EXPORT.toLocaleLowerCase()}`)
+				.replace('-snapshots', '');
 			return result;
 		};
 	};
 
 test.beforeEach(configureSnapshotPath());
 
+// https://playwright.dev/docs/emulation
+test.use({
+	colorScheme: 'light',
+	locale: 'de-DE',
+	isMobile: false,
+	timezoneId: 'Europe/Berlin',
+	viewport: {
+		width: 800,
+		height: 0,
+	},
+});
+
 /**
  * @todo stabilize and re-enable test
  */
 const blocklist = [];
 
-ROUTES.filter((route) => !blocklist.includes(route)).forEach((route) => {
+ROUTES.forEach((options, route) => {
+	if (blocklist.includes(route)) {
+		return;
+	}
 	test(`snapshot for ${route}`, async ({ page }) => {
 		await page.goto(`/#${route}?hideMenus`, { waitUntil: 'networkidle' });
-		await page.setViewportSize({
-			width: 1920,
-			height: 1280,
-		});
+		if (options?.viewportSize) {
+			await page.setViewportSize(options.viewportSize);
+		}
+		if (options?.waitForTimeout) {
+			await page.waitForTimeout(options.waitForTimeout);
+		}
 		await expect(page).toHaveScreenshot({
-			// fullPage: true,
-			maxDiffPixelRatio: 0.03,
+			fullPage: true,
+			maxDiffPixelRatio: 0.025,
+			...options,
 		});
 	});
 });