@rettangoli/vt 0.0.12 → 0.0.14

package/docker/Dockerfile

@@ -0,0 +1,21 @@
+FROM mcr.microsoft.com/playwright:v1.57.0-noble
+
+# Install dependencies for Bun
+RUN apt-get update && apt-get install -y unzip curl && rm -rf /var/lib/apt/lists/*
+
+# Install Bun to system location directly (latest version)
+RUN curl -fsSL https://bun.sh/install | BUN_INSTALL=/usr/local bash
+
+# Install rtgl globally
+RUN bun install -g rtgl@0.0.36
+
+# Copy bun global packages to /usr/local/lib/rtgl (accessible to all users)
+RUN mkdir -p /usr/local/lib/rtgl && \
+    cp -r /root/.bun/install/global /usr/local/lib/rtgl/ && \
+    chmod -R a+rx /usr/local/lib/rtgl/
+
+# Create wrapper script that uses copied packages
+RUN echo "#!/bin/sh" > /usr/local/bin/rtgl && \
+    echo "BUN_INSTALL_LOCKFILE=/usr/local/lib/rtgl/bun/install/cache/bun-install.lock" >> /usr/local/bin/rtgl && \
+    echo "exec bun /usr/local/lib/rtgl/global/node_modules/rtgl/cli.js \"\$@\"" >> /usr/local/bin/rtgl && \
+    chmod +x /usr/local/bin/rtgl

package/docker/build-and-push.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+IMAGE_NAME="playwright-v1.57.0-rtgl-v0.0.36"
+REGISTRY="${REGISTRY:-docker.io}"
+REPO="${REPO:-han4wluc/rtgl}"
+
+FULL_TAG="$REGISTRY/$REPO:$IMAGE_NAME"
+
+docker build -t "$IMAGE_NAME" "$SCRIPT_DIR"
+echo "Built image: $IMAGE_NAME"
+
+docker tag "$IMAGE_NAME" "$FULL_TAG"
+docker push "$FULL_TAG"
+echo "Pushed: $FULL_TAG"

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rettangoli/vt",
-  "version": "0.0.12",
+  "version": "0.0.14",
   "description": "Rettangoli Visual Testing",
   "type": "module",
   "main": "./src/index.js",
@@ -15,8 +15,9 @@
     "commander": "^13.1.0",
     "js-yaml": "^4.1.0",
     "liquidjs": "^10.21.0",
-    "playwright": "^1.52.0",
-    "sharp": "^0.33.0",
+    "pixelmatch": "^7.1.0",
+    "playwright": "1.57.0",
+    "sharp": "^0.34.5",
     "shiki": "^3.3.0"
   }
-}
+}

package/src/cli/accept.js

@@ -1,36 +1,10 @@
-import { existsSync, readdirSync, statSync } from 'node:fs';
-import { rm, cp, mkdir } from 'node:fs/promises';
+import { existsSync, readFileSync } from 'node:fs';
+import { cp, mkdir, rm } from 'node:fs/promises';
 import { join, dirname } from 'node:path';
 
 /**
- * Recursively copy only WebP files from source to destination
- */
-async function copyWebpFiles(sourceDir, destDir) {
-  if (!existsSync(sourceDir)) {
-    return;
-  }
-
-  const items = readdirSync(sourceDir);
-
-  for (const item of items) {
-    const sourcePath = join(sourceDir, item);
-    const destPath = join(destDir, item);
-
-    if (statSync(sourcePath).isDirectory()) {
-      // Recursively copy subdirectories
-      await copyWebpFiles(sourcePath, destPath);
-    } else if (item.endsWith('.webp')) {
-      // Copy WebP files only
-      await mkdir(dirname(destPath), { recursive: true });
-      await cp(sourcePath, destPath);
-      console.log(`Copied: ${sourcePath} -> ${destPath}`);
-    }
-  }
-}
-
-/**
- * Accepts candidate screenshots as the new reference by removing the existing reference
- * directory and copying the candidate directory to reference.
+ * Accepts candidate screenshots as the new reference by copying only files
+ * that have diffs according to report.json.
  */
 async function acceptReference(options = {}) {
   const {
@@ -40,32 +14,48 @@ async function acceptReference(options = {}) {
   const referenceDir = join(vtPath, "reference");
   const siteOutputPath = join(".rettangoli", "vt", "_site");
   const candidateDir = join(siteOutputPath, "candidate");
+  const jsonReportPath = join(".rettangoli", "vt", "report.json");
 
-  console.log('Accepting candidate as new reference...');
-
-  // Check if candidate directory exists
-  if (!existsSync(candidateDir)) {
-    console.error('Error: Candidate directory does not exist!');
+  // Check if report.json exists
+  if (!existsSync(jsonReportPath)) {
+    console.error('Error: report.json not found. Run "rtgl vt report" first.');
     process.exit(1);
   }
 
+  // Read report.json
+  const report = JSON.parse(readFileSync(jsonReportPath, 'utf8'));
+
+  if (report.items.length === 0) {
+    console.log('No differences found in report. Nothing to accept.');
+    return;
+  }
+
+  console.log(`Accepting ${report.items.length} changed files as new reference...`);
+
   try {
-    // Remove reference directory if it exists
-    if (existsSync(referenceDir)) {
-      console.log('Removing existing reference directory...');
-      await rm(referenceDir, { recursive: true, force: true });
-    }
+    for (const item of report.items) {
+      // Skip items that only exist in reference (they should be deleted)
+      if (item.onlyInReference) {
+        const refPath = join(referenceDir, item.referencePath.replace('reference/', ''));
+        if (existsSync(refPath)) {
+          await rm(refPath);
+          console.log(`Removed: ${refPath}`);
+        }
+        continue;
+      }
 
-    // Wait for 100ms to ensure the directory is removed
-    await new Promise((resolve) => {
-      setTimeout(resolve, 100);
-    })
-    
-    // Copy only WebP files from candidate to reference
-    console.log('Copying WebP files from candidate to reference...');
-    await copyWebpFiles(candidateDir, referenceDir);
+      // Copy candidate to reference
+      const candidatePath = join(siteOutputPath, item.candidatePath);
+      const refPath = join(referenceDir, item.candidatePath.replace('candidate/', ''));
+
+      if (existsSync(candidatePath)) {
+        await mkdir(dirname(refPath), { recursive: true });
+        await cp(candidatePath, refPath);
+        console.log(`Copied: ${candidatePath} -> ${refPath}`);
+      }
+    }
 
-    console.log('Done! New reference accepted.');
+    console.log('Done! Changes accepted.');
   } catch (error) {
     console.error('Error:', error.message);
     process.exit(1);

package/src/cli/generate.js

@@ -21,6 +21,8 @@ async function main(options) {
     vtPath = "./vt",
     screenshotWaitTime = 0,
     port = 3001,
+    waitEvent,
+    concurrency = 12,
   } = options;
 
   const specsPath = join(vtPath, "specs");
@@ -100,9 +102,10 @@ async function main(options) {
         filesToScreenshot,
         `http://localhost:${port}`,
         candidatePath,
-        24,
+        concurrency,
         screenshotWaitTime,
         configUrl,
+        waitEvent,
       );
     } finally {
       if (server) {

package/src/cli/report.js

@@ -3,6 +3,8 @@ import path from "path";
 import crypto from "crypto";
 import { Liquid } from "liquidjs";
 import { cp } from "node:fs/promises";
+import pixelmatch from "pixelmatch";
+import sharp from "sharp";
 
 const libraryTemplatesPath = new URL('./templates', import.meta.url).pathname;
 
@@ -70,35 +72,80 @@ async function calculateImageHash(imagePath) {
   }
 }
 
-async function compareImages(artifactPath, goldPath) {
+async function compareImagesMd5(artifactPath, goldPath) {
   try {
     const artifactHash = await calculateImageHash(artifactPath);
     const goldHash = await calculateImageHash(goldPath);
 
     if (artifactHash === null || goldHash === null) {
-      return {
-        equal: false,
-        error: true,
-      };
+      return { equal: false, error: true };
     }
 
-    const equal = artifactHash === goldHash;
-
-    return {
-      equal,
-      error: false,
-    };
+    return { equal: artifactHash === goldHash, error: false };
   } catch (error) {
     console.error("Error comparing images:", error);
-    return {
-      equal: false,
-      error: true,
-      diffBounds: null,
-      diffClusters: null
-    };
+    return { equal: false, error: true };
   }
 }
 
+async function compareImagesPixelmatch(artifactPath, goldPath, diffPath, options = {}) {
+  const { colorThreshold = 0.1, diffThreshold = 0.3 } = options;
+
+  try {
+    // Load images and convert to raw RGBA using sharp
+    const [artifactData, goldData] = await Promise.all([
+      sharp(artifactPath).ensureAlpha().raw().toBuffer({ resolveWithObject: true }),
+      sharp(goldPath).ensureAlpha().raw().toBuffer({ resolveWithObject: true }),
+    ]);
+
+    const { width, height } = artifactData.info;
+    const goldWidth = goldData.info.width;
+    const goldHeight = goldData.info.height;
+    const totalPixels = width * height;
+
+    // If dimensions don't match, images are different
+    if (width !== goldWidth || height !== goldHeight) {
+      return { equal: false, error: false, diffPixels: -1, totalPixels, similarity: 0 };
+    }
+
+    // Create diff image buffer
+    const diffBuffer = Buffer.alloc(totalPixels * 4);
+
+    const diffPixels = pixelmatch(
+      artifactData.data,
+      goldData.data,
+      diffBuffer,
+      width,
+      height,
+      { threshold: colorThreshold }
+    );
+
+    const diffPercent = (diffPixels / totalPixels) * 100;
+    const similarity = (100 - diffPercent).toFixed(2);
+    const equal = diffPercent < diffThreshold;
+
+    // Save diff image if not equal
+    if (!equal && diffPath) {
+      fs.mkdirSync(path.dirname(diffPath), { recursive: true });
+      await sharp(diffBuffer, { raw: { width, height, channels: 4 } })
+        .png()
+        .toFile(diffPath);
+    }
+
+    return { equal, error: false, diffPixels, totalPixels, similarity };
+  } catch (error) {
+    console.error("Error comparing images with pixelmatch:", error);
+    return { equal: false, error: true };
+  }
+}
+
+async function compareImages(artifactPath, goldPath, method = 'pixelmatch', diffPath = null, options = {}) {
+  if (method === 'md5') {
+    return compareImagesMd5(artifactPath, goldPath);
+  }
+  return compareImagesPixelmatch(artifactPath, goldPath, diffPath, options);
+}
+
 async function generateReport({ results, templatePath, outputPath }) {
   try {
     // Read the template file
@@ -119,20 +166,37 @@ async function generateReport({ results, templatePath, outputPath }) {
 }
 
 async function main(options = {}) {
-  const { vtPath = "./vt" } = options;
+  const {
+    vtPath = "./vt",
+    compareMethod = 'pixelmatch',
+    colorThreshold = 0.1,
+    diffThreshold = 0.3,
+  } = options;
 
   const siteOutputPath = path.join(".rettangoli", "vt", "_site");
   const candidateDir = path.join(siteOutputPath, "candidate");
+  const diffDir = path.join(siteOutputPath, "diff");
   const originalReferenceDir = path.join(vtPath, "reference");
   const siteReferenceDir = path.join(siteOutputPath, "reference");
   const templatePath = path.join(libraryTemplatesPath, "report.html");
   const outputPath = path.join(siteOutputPath, "report.html");
+  const jsonReportPath = path.join(".rettangoli", "vt", "report.json");
+
+  console.log(`Comparison method: ${compareMethod}`);
+  if (compareMethod === 'pixelmatch') {
+    console.log(`  color threshold: ${colorThreshold}, diff threshold: ${diffThreshold}%`);
+  }
 
   if (!fs.existsSync(originalReferenceDir)) {
     console.log("Reference directory does not exist, creating it...");
     fs.mkdirSync(originalReferenceDir, { recursive: true });
   }
 
+  // Create diff directory for diffs
+  if (compareMethod === 'pixelmatch' && !fs.existsSync(diffDir)) {
+    fs.mkdirSync(diffDir, { recursive: true });
+  }
+
   // Copy reference directory to _site for web access
   if (fs.existsSync(originalReferenceDir)) {
     console.log("Copying reference directory to _site...");
@@ -168,6 +232,7 @@ async function main(options = {}) {
       const candidatePath = path.join(candidateDir, relativePath);
       const referencePath = path.join(originalReferenceDir, relativePath);
       const siteReferencePath = path.join(siteReferenceDir, relativePath);
+      const diffPath = path.join(diffDir, relativePath.replace('.webp', '-diff.png'));
 
       const candidateExists = fs.existsSync(candidatePath);
       const referenceExists = fs.existsSync(referencePath);
@@ -177,12 +242,28 @@ async function main(options = {}) {
 
       let equal = true;
       let error = false;
+      let similarity = null;
+      let diffPixels = null;
 
       // Compare images if both exist
       if (candidateExists && referenceExists) {
-        const comparison = await compareImages(candidatePath, referencePath);
+        // Ensure diff directory exists
+        const diffDirPath = path.dirname(diffPath);
+        if (!fs.existsSync(diffDirPath)) {
+          fs.mkdirSync(diffDirPath, { recursive: true });
+        }
+
+        const comparison = await compareImages(
+          candidatePath,
+          referencePath,
+          compareMethod,
+          diffPath,
+          { colorThreshold, diffThreshold }
+        );
         equal = comparison.equal;
         error = comparison.error;
+        similarity = comparison.similarity;
+        diffPixels = comparison.diffPixels;
       } else {
         equal = false; // If one file is missing, they're not equal
       }
@@ -193,6 +274,8 @@ async function main(options = {}) {
           referencePath: referenceExists ? siteReferencePath : null, // Use site reference path for HTML report
           path: relativePath,
           equal: candidateExists && referenceExists ? equal : false,
+          similarity,
+          diffPixels,
           onlyInCandidate: candidateExists && !referenceExists,
           onlyInReference: !candidateExists && referenceExists,
         });
@@ -213,6 +296,8 @@ async function main(options = {}) {
             ? path.relative(siteOutputPath, result.referencePath)
             : null,
           equal: result.equal,
+          similarity: result.similarity,
+          diffPixels: result.diffPixels,
           onlyInCandidate: result.onlyInCandidate,
           onlyInReference: result.onlyInReference,
         };
@@ -223,6 +308,8 @@ async function main(options = {}) {
         candidatePath: item.candidatePath,
         referencePath: item.referencePath,
         equal: item.equal,
+        similarity: item.similarity ? `${item.similarity}%` : null,
+        diffPixels: item.diffPixels,
       };
       console.log(JSON.stringify(logData, null, 2));
     });
@@ -238,6 +325,25 @@ async function main(options = {}) {
       templatePath,
       outputPath,
     });
+
+    // Write JSON report
+    const jsonReport = {
+      timestamp: new Date().toISOString(),
+      total: results.length,
+      mismatched: mismatchingItems.length,
+      items: mismatchingItems.map(item => ({
+        path: item.candidatePath || item.referencePath,
+        candidatePath: item.candidatePath,
+        referencePath: item.referencePath,
+        equal: item.equal,
+        similarity: item.similarity,
+        onlyInCandidate: item.onlyInCandidate,
+        onlyInReference: item.onlyInReference,
+      })),
+    };
+    fs.writeFileSync(jsonReportPath, JSON.stringify(jsonReport, null, 2));
+    console.log(`JSON report written to ${jsonReportPath}`);
+
     if(mismatchingItems.length > 0){
       console.error("Error: there are more than 0 mismatching item.")
       process.exit(1);

package/src/common.js

@@ -247,8 +247,9 @@ async function takeScreenshots(
   serverUrl,
   screenshotsDir,
   concurrency = 8,
-  waitTime = 0,
+  waitTime = 10,
   configUrl = undefined,
+  waitEvent = undefined,
 ) {
   // Ensure screenshots directory exists
   ensureDirectoryExists(screenshotsDir);
@@ -263,78 +264,155 @@ async function takeScreenshots(
     const screenshotPath = join(screenshotsDir, `${finalPath}.webp`);
     ensureDirectoryExists(dirname(screenshotPath));
 
-    await page.screenshot({ path: tempPngPath, fullPage: true });
-    await sharp(tempPngPath).webp({ quality: 85 }).toFile(screenshotPath);
+    // Check if custom screenshot function is available
+    const hasCustomScreenshot = await page.evaluate(() => typeof window.takeVtScreenshotBase64 === 'function');
 
-    if (existsSync(tempPngPath)) {
-      unlinkSync(tempPngPath);
+    if (hasCustomScreenshot) {
+      // Use custom screenshot function (useful for canvas-based apps)
+      let base64Data = await page.evaluate(() => window.takeVtScreenshotBase64());
+      // Strip data URL prefix if present (e.g., "data:image/png;base64,")
+      if (base64Data.includes(',')) {
+        base64Data = base64Data.split(',')[1];
+      }
+      const pngBuffer = Buffer.from(base64Data, 'base64');
+      await sharp(pngBuffer).webp({ quality: 85 }).toFile(screenshotPath);
+    } else {
+      // Use Playwright's built-in screenshot
+      await page.screenshot({ path: tempPngPath, fullPage: true });
+      await sharp(tempPngPath).webp({ quality: 85 }).toFile(screenshotPath);
+
+      if (existsSync(tempPngPath)) {
+        unlinkSync(tempPngPath);
+      }
     }
     return screenshotPath;
   };
 
-  try {
-    const files = [...generatedFiles];
-    const total = files.length;
-    let completed = 0;
+  const processFile = async (file, browser, serverUrl, screenshotsDir, waitTime, configUrl, waitEvent, takeAndSaveScreenshot) => {
+    const context = await browser.newContext();
+    const page = await context.newPage();
 
-    // Process files in batches based on concurrency
-    while (files.length > 0) {
-      const batch = files.splice(0, concurrency);
-      const batchPromises = batch.map(async (file) => {
-        // Create a new context and page for each file (for parallelism)
-        const context = await browser.newContext();
-        const page = await context.newPage();
-
-        try {
-          const envVars = {};
-          for (const [key, value] of Object.entries(process.env)) {
-            if (key.startsWith('RTGL_VT_')) {
-              envVars[key] = value;
-            }
-          }
+    try {
+      const envVars = {};
+      for (const [key, value] of Object.entries(process.env)) {
+        if (key.startsWith('RTGL_VT_')) {
+          envVars[key] = value;
+        }
+      }
 
-          if (Object.keys(envVars).length > 0) {
-            await page.addInitScript((vars) => {
-              Object.assign(window, vars);
-            }, envVars);
-          }
+      if (Object.keys(envVars).length > 0) {
+        await page.addInitScript((vars) => {
+          Object.assign(window, vars);
+        }, envVars);
+      }
+
+      // If using custom wait event, set up listener before navigation
+      if (waitEvent) {
+        await page.addInitScript((eventName) => {
+          window.__vtReadyFired = false;
+          window.addEventListener(eventName, () => {
+            window.__vtReadyFired = true;
+          }, { once: true });
+        }, waitEvent);
+      }
 
-          const frontMatterUrl = file.frontMatter?.url;
-          const constructedUrl = convertToHtmlExtension(`${serverUrl}/candidate/${file.path.replace(/\\/g, "/")}`);
-          const url = frontMatterUrl ?? configUrl ?? constructedUrl;
-          const fileUrl = url.startsWith("http") ? url : new URL(url, serverUrl).href;
+      const frontMatterUrl = file.frontMatter?.url;
+      const constructedUrl = convertToHtmlExtension(`${serverUrl}/candidate/${file.path.replace(/\\/g, "/")}`);
+      const url = frontMatterUrl ?? configUrl ?? constructedUrl;
+      const fileUrl = url.startsWith("http") ? url : new URL(url, serverUrl).href;
+
+      console.log(`Navigating to ${fileUrl}`);
+
+      if (waitEvent) {
+        // Navigate and wait for custom event
+        await page.goto(fileUrl, { waitUntil: "load" });
+        console.log(`Waiting for custom event: ${waitEvent}`);
+        await page.waitForFunction(() => window.__vtReadyFired === true, { timeout: 30000 });
+      } else {
+        // Default: wait for network idle
+        await page.goto(fileUrl, { waitUntil: "networkidle" });
+      }
 
-          console.log(`Navigating to ${fileUrl}`);
-          await page.goto(fileUrl, { waitUntil: "networkidle" });
-          if (waitTime > 0) {
-            await page.waitForTimeout(waitTime);
+      // Normalize font rendering for consistent screenshots
+      await page.addStyleTag({
+        content: `
+          * {
+            -webkit-font-smoothing: antialiased !important;
+            -moz-osx-font-smoothing: grayscale !important;
+            text-rendering: geometricPrecision !important;
           }
-          const baseName = removeExtension(file.path);
+        `
+      });
+
+      if (waitTime > 0) {
+        await page.waitForTimeout(waitTime);
+      }
+      const baseName = removeExtension(file.path);
 
-          const initialScreenshotPath = await takeAndSaveScreenshot(page, baseName);
-          console.log(`Screenshot saved: ${initialScreenshotPath}`);
+      const initialScreenshotPath = await takeAndSaveScreenshot(page, baseName);
+      console.log(`Screenshot saved: ${initialScreenshotPath}`);
 
-          const stepContext = {
-            baseName,
-            takeAndSaveScreenshot,
-          };
-          const stepsExecutor = createSteps(page, stepContext);
+      const stepContext = {
+        baseName,
+        takeAndSaveScreenshot,
+      };
+      const stepsExecutor = createSteps(page, stepContext);
 
-          for (const step of file.frontMatter?.steps || []) {
-            await stepsExecutor.executeStep(step);
+      for (const step of file.frontMatter?.steps || []) {
+        await stepsExecutor.executeStep(step);
+      }
+      return { success: true, file };
+    } finally {
+      await context.close();
+    }
+  };
+
+  try {
+    const total = generatedFiles.length;
+    let completed = 0;
+    let filesToProcess = [...generatedFiles];
+    const maxRetries = 3;
+
+    for (let attempt = 1; attempt <= maxRetries && filesToProcess.length > 0; attempt++) {
+      if (attempt > 1) {
+        console.log(`\nRetry attempt ${attempt}/${maxRetries} for ${filesToProcess.length} failed files...`);
+      }
+
+      const failedFiles = [];
+
+      // Process files in batches based on concurrency
+      while (filesToProcess.length > 0) {
+        const batch = filesToProcess.splice(0, concurrency);
+        const batchPromises = batch.map((file) =>
+          processFile(file, browser, serverUrl, screenshotsDir, waitTime, configUrl, waitEvent, takeAndSaveScreenshot)
+            .then(() => {
+              completed++;
+              console.log(`Finished processing ${file.path} (${completed}/${total})`);
+              return { success: true, file };
+            })
+            .catch((error) => {
+              console.error(`Error processing ${file.path}:`, error.message);
+              return { success: false, file };
+            })
+        );
+
+        const results = await Promise.allSettled(batchPromises);
+        for (const result of results) {
+          if (result.status === 'fulfilled' && !result.value.success) {
+            failedFiles.push(result.value.file);
+          } else if (result.status === 'rejected') {
+            // This shouldn't happen since we catch errors above, but just in case
+            console.error('Unexpected rejection:', result.reason);
           }
-          completed++;
-          console.log(`Finished processing ${file.path} (${completed}/${total})`);
-        } catch (error) {
-          console.error(`Error processing instructions for ${file.path}:`, error);
-        } finally {
-          // Close the context when done
-          await context.close();
         }
-      });
+      }
+
+      filesToProcess = failedFiles;
+    }
 
-      // Wait for current batch to complete before processing next batch
-      await Promise.all(batchPromises);
+    if (filesToProcess.length > 0) {
+      console.error(`\nFailed to process ${filesToProcess.length} files after ${maxRetries} attempts:`);
+      filesToProcess.forEach((file) => console.error(`  - ${file.path}`));
     }
   } catch (error) {
     console.error("Error taking screenshots:", error);
@@ -437,4 +515,4 @@ export {
   takeScreenshots,
   generateOverview,
   readYaml,
-};
+};

package/src/createSteps.js

@@ -22,6 +22,16 @@ async function customEvent(page, args) {
 
 async function goto(page, args) {
   await page.goto(args[0], { waitUntil: "networkidle" });
+  // Normalize font rendering for consistent screenshots
+  await page.addStyleTag({
+    content: `
+      * {
+        -webkit-font-smoothing: antialiased !important;
+        -moz-osx-font-smoothing: grayscale !important;
+        text-rendering: geometricPrecision !important;
+      }
+    `
+  });
 }
 
 async function keypress(page, args) {