@brightspace-ui/testing 1.38.2 → 1.39.0

package/package.json

@@ -2,7 +2,7 @@
   "name": "@brightspace-ui/testing",
   "description": "Utilities for testing front-end components and applications",
   "type": "module",
-  "version": "1.38.2",
+  "version": "1.39.0",
   "repository": "https://github.com/BrightspaceUI/testing.git",
   "scripts": {
     "lint": "npm run lint:eslint && npm run lint:style",

package/src/browser/vdiff.js

@@ -89,12 +89,11 @@ async function ScreenshotAndCompare(opts = {}) {
 		({ rect, fullPage } = findLargestRect(elemsToInclude, margin));
 	}
 	const slowDuration = this.test.slow();
+	const altTests = [];
+	if (opts.allColorModes) altTests.push('dark');
 
-	let result = await executeServerCommand('brightspace-visual-diff-compare', { name, fullPage, rect, slowDuration });
-	if (result.resizeRequired) {
-		this.test.timeout(0);
-		result = await executeServerCommand('brightspace-visual-diff-compare-resize', { name });
-	}
+	this.test.timeout(0);
+	const result = await executeServerCommand('brightspace-visual-diff-compare', { altTests, name, fullPage, rect, slowDuration });
 
 	if (window.d2lTest) document.documentElement.classList.remove('screenshot');
 

package/src/server/visual-diff-info.js

@@ -8,9 +8,9 @@ export class TestInfoManager {
 		return testInfoMap.get(this.key);
 	}
 
-	set(testInfo) {
+	set(testInfo, alt = 'default') {
 		if (testInfoMap.has(this.key)) {
-			const info = this.get();
+			const info = this.get()?.[alt] || {};
 			testInfo.slowDuration = info.slowDuration;
 
 			if (info.golden || testInfo.golden) {
@@ -21,7 +21,8 @@ export class TestInfoManager {
 			}
 			testInfo.diff = testInfo.diff || info.diff;
 		}
-		testInfoMap.set(this.key, testInfo);
+		const tests = testInfoMap.get(this.key) || {};
+		testInfoMap.set(this.key, { ...tests, [alt]: testInfo });
 	}
 }
 

package/src/server/visual-diff-plugin.js

@@ -9,6 +9,17 @@ import { TestInfoManager } from './visual-diff-info.js';
 const isCI = !!env['CI'];
 const DEFAULT_TOLERANCE = 0; // TODO: Support tolerance override?
 
+const ALT_TESTS = {
+	dark: {
+		set() {
+			document.documentElement.setAttribute('data-color-mode', 'dark');
+		},
+		reset() {
+			document.documentElement.removeAttribute('data-color-mode');
+		}
+	}
+};
+
 async function checkFileExists(fileName) {
 	try {
 		await access(fileName, constants.F_OK);
@@ -67,6 +78,45 @@ function getResizedPng(xName, yName, original, newSize) {
 	return resized;
 }
 
+let bestPosition = null;
+async function getBestResizePng(screenshotImage, goldenImage) {
+	const newWidth = Math.max(screenshotImage.width, goldenImage.width);
+	const newHeight = Math.max(screenshotImage.height, goldenImage.height);
+	const newSize = { width: newWidth, height: newHeight };
+
+	if (bestPosition !== null) {
+		const [y, x] = bestPosition.split('-');
+		const screenshot = getResizedPng(x, y, screenshotImage, newSize);
+		const golden = getResizedPng(x, y, goldenImage, newSize);
+		const { diff, pixelsDiff } = getPixelsDiff(screenshot, golden);
+		return { screenshot, golden, diff, pixelsDiff };
+	}
+
+	const newScreenshots = await createComparisonPNGs(screenshotImage, newSize);
+	const newGoldens = await createComparisonPNGs(goldenImage, newSize);
+
+	let bestIndex = -1;
+	let bestDiffImage = null;
+	let bestPixelsDiff = Number.MAX_SAFE_INTEGER;
+	for (let i = 0; i < newScreenshots.length; i++) {
+		const { diff, pixelsDiff } = getPixelsDiff(newScreenshots[i].png, newGoldens[i].png);
+
+		if (pixelsDiff < bestPixelsDiff) {
+			bestIndex = i;
+			bestDiffImage = diff;
+			bestPixelsDiff = pixelsDiff;
+		}
+	}
+
+	bestPosition = newScreenshots[bestIndex].position;
+	return {
+		screenshot: newScreenshots[bestIndex].png,
+		golden: newGoldens[bestIndex].png,
+		diff: bestDiffImage,
+		pixelsDiff: bestPixelsDiff
+	};
+}
+
 async function createComparisonPNGs(original, newSize) {
 	const resizedPNGs = [];
 	[ 'top', 'center', 'bottom' ].forEach(y => {
@@ -145,156 +195,156 @@ export function visualDiff({ updateGoldens = false, runSubset = false } = {}) {
 			const { dir, newName } = extractTestPartsFromName(payload.name);
 			const testPath = dirname(session.testFile).replace(rootDir, '');
 			const newPath = join(rootDir, PATHS.VDIFF_ROOT, testPath, dir);
-			const goldenFileName = `${join(newPath, PATHS.GOLDEN, browser, newName)}.png`;
-			const passFileName = `${join(newPath, PATHS.PASS, browser, newName)}.png`;
+			const goldenFile = join(newPath, PATHS.GOLDEN, browser, newName);
+			const passFile = join(newPath, PATHS.PASS, browser, newName);
 			const screenshotFile = join(newPath, PATHS.FAIL, browser, newName);
-			const screenshotFileName = `${screenshotFile}.png`;
-
-			if (command === 'brightspace-visual-diff-compare') {
-				if (!isCI) { // CI will be a fresh .vdiff folder each time and only one run
-					if (session.testRun !== currentRun) {
-						currentRun = session.testRun;
-						clearedDirs.clear();
-					}
+			function getFileName(base, alt = 'default') {
+				return `${base}${alt === 'default' ? '' : `.${alt}`}.png`;
+			}
+			const tests = ['default', ...(payload.altTests || [])];
+			const rootLength = join(rootDir, PATHS.VDIFF_ROOT).length + 1;
+
+			if (command !== 'brightspace-visual-diff-compare') return;
+			if (!isCI) { // CI will be a fresh .vdiff folder each time and only one run
+				if (session.testRun !== currentRun) {
+					currentRun = session.testRun;
+					clearedDirs.clear();
+				}
 
-					if (runSubset || currentRun > 0) {
-						if (!clearedDirs.has(newPath)) {
-							clearedDirs.set(newPath, clearDir(updateGoldens, newPath));
-						}
-						await clearedDirs.get(newPath);
+				if (runSubset || currentRun > 0) {
+					if (!clearedDirs.has(newPath)) {
+						clearedDirs.set(newPath, clearDir(updateGoldens, newPath));
 					}
+					await clearedDirs.get(newPath);
 				}
+			}
 
-				const screenshotOpts = {
-					animations: 'disabled'
-				};
+			const screenshotOpts = {
+				animations: 'disabled'
+			};
 
-				if (payload.fullPage) screenshotOpts.fullPage = true;
-				if (payload.rect) screenshotOpts.clip = payload.rect;
+			if (payload.fullPage) screenshotOpts.fullPage = true;
+			if (payload.rect) screenshotOpts.clip = payload.rect;
 
-				async function takeScreenshot() {
-					const path = updateGoldens ? goldenFileName : screenshotFileName;
-					const page = session.browser.getPage(session.id);
-					return await page.screenshot({ path, ...screenshotOpts });
-				}
+			const page = session.browser.getPage(session.id);
+			async function takeScreenshot(alt = 'default') {
+				const path = getFileName(updateGoldens ? goldenFile : screenshotFile, alt);
+				if (ALT_TESTS[alt]?.set) page.evaluate(ALT_TESTS[alt].set);
+				const screenshot = await page.screenshot({ path, ...screenshotOpts });
+				if (ALT_TESTS[alt]?.reset) page.evaluate(ALT_TESTS[alt].reset);
+				return screenshot;
+			}
 
-				const screenshotFileBuffer = await takeScreenshot();
+			const screenshotFileBuffers = {};
+			for (const test of tests) screenshotFileBuffers[test] = await takeScreenshot(test);
 
-				if (updateGoldens) {
-					return { pass: true };
-				}
+			if (updateGoldens) {
+				return { pass: true };
+			}
 
-				const rootLength = join(rootDir, PATHS.VDIFF_ROOT).length + 1;
+			async function runCompareTest(alt = 'default') {
+				const screenshotFileName = getFileName(screenshotFile, alt);
+				const goldenFileName = getFileName(goldenFile, alt);
+				const passFileName = getFileName(passFile, alt);
 
-				async function runCompareTest() {
-					const screenshotImage = PNG.sync.read(screenshotFileBuffer);
-					infoManager.set({
-						slowDuration: payload.slowDuration,
-						new: {
-							height: screenshotImage.height,
-							path: screenshotFileName.substring(rootLength),
-							width: screenshotImage.width
-						}
-					});
-
-					const goldenExists = await checkFileExists(goldenFileName);
-					if (!goldenExists) {
-						return { pass: false, message: 'No golden exists. Use the "--golden" CLI flag to re-run and re-generate goldens.' };
+				const screenshotImage = PNG.sync.read(screenshotFileBuffers[alt]);
+				infoManager.set({
+					slowDuration: payload.slowDuration,
+					new: {
+						height: screenshotImage.height,
+						path: screenshotFileName.substring(rootLength),
+						width: screenshotImage.width
 					}
+				}, alt);
 
-					const goldenFileBuffer = await readFile(goldenFileName);
-					const goldenImage = PNG.sync.read(goldenFileBuffer);
-					infoManager.set({
-						golden: {
-							height: goldenImage.height,
-							path: goldenFileName.substring(rootLength),
-							width: goldenImage.width
-						}
-					});
-
-					if (screenshotImage.width === goldenImage.width && screenshotImage.height === goldenImage.height) {
-						const goldenSize = (await stat(goldenFileName)).size;
-						const screenshotSize = (await stat(screenshotFileName)).size;
-						if (goldenSize === screenshotSize && screenshotFileBuffer.equals(goldenFileBuffer)) {
-							const success = await tryMoveFile(screenshotFileName, passFileName);
-							if (!success) return { pass: false, message: 'Problem moving file to "pass" directory.' };
-							infoManager.set({
-								new: {
-									path: passFileName.substring(rootLength)
-								}
-							});
-							return { pass: true };
-						}
-
-						const { diff, pixelsDiff } = getPixelsDiff(screenshotImage, goldenImage);
-
-						if (pixelsDiff !== 0) {
-							infoManager.set({
-								diff: `${screenshotFile.substring(rootLength)}-diff.png`,
-								pixelsDiff
-							});
-							await writeFile(`${screenshotFile}-diff.png`, PNG.sync.write(diff));
-							return { pass: false, message: `Image does not match golden. ${pixelsDiff} pixels are different.` };
-						} else {
-							infoManager.set({
-								golden: {
-									byteSize: goldenSize
-								},
-								new: {
-									byteSize: screenshotSize
-								},
-								pixelsDiff
-							});
-							return { pass: false, message: 'Image diff is clean but the images do not have the same bytes.' };
-						}
-					} else {
-						return { resizeRequired: true };
-					}
-				}
-				return await runCompareTest();
-			} else if (command === 'brightspace-visual-diff-compare-resize') {
-				const screenshotImage = PNG.sync.read(await readFile(screenshotFileName));
-				const goldenImage = PNG.sync.read(await readFile(goldenFileName));
-
-				const newWidth = Math.max(screenshotImage.width, goldenImage.width);
-				const newHeight = Math.max(screenshotImage.height, goldenImage.height);
-				const newSize = { width: newWidth, height: newHeight };
-
-				const newScreenshots = await createComparisonPNGs(screenshotImage, newSize);
-				const newGoldens = await createComparisonPNGs(goldenImage, newSize);
-
-				let bestIndex = -1;
-				let bestDiffImage = null;
-				let bestPixelsDiff = Number.MAX_SAFE_INTEGER;
-				for (let i = 0; i < newScreenshots.length; i++) {
-					const { diff, pixelsDiff } = getPixelsDiff(newScreenshots[i].png, newGoldens[i].png);
-
-					if (pixelsDiff < bestPixelsDiff) {
-						bestIndex = i;
-						bestDiffImage = diff;
-						bestPixelsDiff = pixelsDiff;
-					}
+				const goldenExists = await checkFileExists(goldenFileName);
+				if (!goldenExists) {
+					return { pass: false, message: 'No golden exists. Use the "--golden" CLI flag to re-run and re-generate goldens.' };
 				}
 
-				await writeFile(`${screenshotFile}-resized-screenshot.png`, PNG.sync.write(newScreenshots[bestIndex].png));
-				await writeFile(`${screenshotFile}-resized-golden.png`, PNG.sync.write(newGoldens[bestIndex].png));
-				await writeFile(`${screenshotFile}-diff.png`, PNG.sync.write(bestDiffImage));
-
-				const rootLength = join(rootDir, PATHS.VDIFF_ROOT).length + 1;
+				const goldenFileBuffer = await readFile(goldenFileName);
+				const goldenImage = PNG.sync.read(goldenFileBuffer);
 				infoManager.set({
-					diff: `${screenshotFile.substring(rootLength)}-diff.png`,
 					golden: {
-						path: `${screenshotFile.substring(rootLength)}-resized-golden.png`
-					},
-					new: {
-						path: `${screenshotFile.substring(rootLength)}-resized-screenshot.png`
-					},
-					pixelsDiff: bestPixelsDiff
-				});
+						height: goldenImage.height,
+						path: goldenFileName.substring(rootLength),
+						width: goldenImage.width
+					}
+				}, alt);
+
+				if (screenshotImage.width === goldenImage.width && screenshotImage.height === goldenImage.height) {
+					const goldenSize = (await stat(goldenFileName)).size;
+					const screenshotSize = (await stat(screenshotFileName)).size;
+					if (goldenSize === screenshotSize && screenshotFileBuffers[alt].equals(goldenFileBuffer)) {
+						const success = await tryMoveFile(screenshotFileName, passFileName);
+						if (!success) return { pass: false, message: 'Problem moving file to "pass" directory.' };
+						infoManager.set({
+							new: {
+								path: passFileName.substring(rootLength)
+							}
+						}, alt);
+						return { pass: true };
+					}
+
+					const { diff, pixelsDiff } = getPixelsDiff(screenshotImage, goldenImage);
+
+					if (pixelsDiff !== 0) {
+						infoManager.set({
+							diff: getFileName(`${screenshotFile}-diff`, alt).substring(rootLength),
+							pixelsDiff
+						}, alt);
+						await writeFile(getFileName(`${screenshotFile}-diff`, alt), PNG.sync.write(diff));
+						return { pass: false, message: `Image does not match golden. ${pixelsDiff} pixels are different.` };
+					} else {
+						infoManager.set({
+							golden: {
+								byteSize: goldenSize
+							},
+							new: {
+								byteSize: screenshotSize
+							},
+							pixelsDiff
+						}, alt);
+						return { pass: false, message: 'Image diff is clean but the images do not have the same bytes.' };
+					}
+				} else {
+					const { screenshot, golden, diff, pixelsDiff } = await getBestResizePng(screenshotImage, goldenImage);
+
+					const screenshotResizedPath = getFileName(`${screenshotFile}-resized-screenshot`, alt);
+					const goldenResizedPath = getFileName(`${screenshotFile}-resized-golden`, alt);
+					const diffPath = getFileName(`${screenshotFile}-diff`, alt);
+					await writeFile(screenshotResizedPath, PNG.sync.write(screenshot));
+					await writeFile(goldenResizedPath, PNG.sync.write(golden));
+					await writeFile(diffPath, PNG.sync.write(diff));
+					infoManager.set({
+						diff: diffPath.substring(rootLength),
+						golden: {
+							path: goldenResizedPath.substring(rootLength)
+						},
+						new: {
+							path: screenshotResizedPath.substring(rootLength)
+						},
+						pixelsDiff
+					}, alt);
 
-				return { pass: false, message: `Images are not the same size. When resized, ${bestPixelsDiff} pixels are different.` };
+					return { pass: false, message: `Images are not the same size. When resized, ${pixelsDiff} pixels are different.` };
+				}
 			}
 
+			bestPosition = null; // reset best position
+			let pass = true;
+			let errors = '';
+			const testResults = {};
+			for (const test of tests) {
+				testResults[test] = await runCompareTest(test);
+				if (!testResults[test].pass) {
+					pass = false;
+					errors += `\n(${test}) ${testResults[test].message}`;
+				}
+			}
+			if (pass) return { pass: true };
+			else if (tests.length === 1) return testResults[tests[0]];
+			else return { pass: false, message: `One or more tests failed:${errors}` };
 		}
 	};
 }

package/src/server/visual-diff-reporter.js

@@ -85,39 +85,43 @@ function flattenResults(session, browserData, fileData) {
 
 	function collectTests(prefix, tests) {
 		tests.forEach(t => {
-			const testName = `${prefix}${t.name}`;
+			let testName = `${prefix}${t.name}`;
 			const testKey = testName.replaceAll(' > ', ' ');
-			const info = getTestInfo(session, testKey);
+			const allTestInfo = getTestInfo(session, testKey);
 
 			// tests missing info but with no error were skipped via grep, so exclude them
-			if (!info && !t.error) return;
-
-			if (!fileData.tests.has(testName)) {
-				fileData.tests.set(testName, {
-					name: testName,
-					numByteDiff: 0,
-					numFailed: 0,
-					results: []
-				});
-			}
-			const testData = fileData.tests.get(testName);
-			const bytediff = !t.passed && info?.diff === undefined && info?.pixelsDiff === 0;
-			if (!t.passed) {
-				if (bytediff) {
-					browserData.numByteDiff++;
-					testData.numByteDiff++;
+			if (!allTestInfo && !t.error) return;
+
+			for (const test in allTestInfo) {
+				const info = allTestInfo[test];
+				if (test !== 'default') testName = `${prefix}${t.name} (${test})`;
+				if (!fileData.tests.has(testName)) {
+					fileData.tests.set(testName, {
+						name: testName,
+						numByteDiff: 0,
+						numFailed: 0,
+						results: []
+					});
+				}
+				const testData = fileData.tests.get(testName);
+				const bytediff = !t.passed && info?.diff === undefined && info?.pixelsDiff === 0;
+				if (!t.passed) {
+					if (bytediff) {
+						browserData.numByteDiff++;
+						testData.numByteDiff++;
+					}
+					browserData.numFailed++;
+					testData.numFailed++;
 				}
-				browserData.numFailed++;
-				testData.numFailed++;
+				testData.results.push({
+					name: browserData.name,
+					duration: t.duration,
+					error: t.error?.message,
+					passed: t.passed,
+					bytediff,
+					info: info
+				});
 			}
-			testData.results.push({
-				name: browserData.name,
-				duration: t.duration,
-				error: t.error?.message,
-				passed: t.passed,
-				bytediff,
-				info: info
-			});
 		});
 	}